Fix help types, dump skip_default, kwargs pop and shtab redraw

CHANGELOG.rst

@@ -150,6 +150,27 @@ Fixed
   decorators that mark a class as deprecated or experimental. The parameters
   were resolved from the wrapper instead of from ``__init__`` (`#953
   <https://github.com/mauvilsa/jsonargparse/pull/953>`__).
+- Types shown in the help being mangled when they contain a dot, since the
+  stripping of module names was done with a regex over the entire type string.
+  Affected floats, literals and ellipsis (`#954
+  <https://github.com/mauvilsa/jsonargparse/pull/954>`__).
+- ``None`` in a PEP 604 union shown as is in the help, e.g. ``date | None``,
+  instead of as ``date | null``, which is what parsing requires (`#954
+  <https://github.com/mauvilsa/jsonargparse/pull/954>`__).
+- ``dump`` with ``skip_default=True``, i.e. ``--print_config=skip_default``,
+  failing with ``AttributeError: 'NoneType' object has no attribute 'get'`` when
+  a subclass spec is given for an argument whose default is ``None``, e.g. an
+  ``Optional[SomeClass]`` parameter. Now the ``class_path`` is kept in the dump
+  and only the ``init_args`` that are defaults are removed (`#954
+  <https://github.com/mauvilsa/jsonargparse/pull/954>`__).
+- Parameters popped or gotten from ``**kwargs`` disappearing when the value is
+  then forwarded explicitly as a keyword, e.g. ``x = kwargs.pop("x", None)``
+  followed by ``super().__init__(x=x, **kwargs)`` (`#954
+  <https://github.com/mauvilsa/jsonargparse/pull/954>`__).
+- ``shtab`` bash completion not redrawing the prompt after printing the type
+  guidance message when there are zero completions, leaving the cursor on an
+  empty line, observed since bash 5.3 (`#954
+  <https://github.com/mauvilsa/jsonargparse/pull/954>`__).
 
 Changed
 ^^^^^^^

jsonargparse/_completions.py

@@ -257,7 +257,12 @@ def shtab_prepare_action(action, parser) -> None:
 
 # "{prog}" is a placeholder replaced with the normalized prog name, see get_shtab_script.
 bash_compgen_typehint_name = "_jsonargparse_{prog}_compgen_typehint"
+# When there are zero completions only a message is printed, so readline doesn't redraw the
+# prompt. A SIGWINCH doesn't help since readline redraws only if the terminal size changed.
+# Thus, ask the terminal for a device status report (\\e[5n) and bind its reply (\\e[0n) to
+# redraw-current-line, making readline itself redraw once the completion function returns.
 bash_compgen_typehint = """
+[[ $- == *i* ]] && bind '"\\e[0n": redraw-current-line' 2>/dev/null
 %(name)s() {
   local CHOICES="$1" WORD="$2" MESSAGE="$3" REQUIRE_PREFIX="$4" TOTAL="$5"
   local IFS=$'\\n'  # choices may contain spaces, so split matches on newline only
@@ -274,7 +279,7 @@ def shtab_prepare_action(action, parser) -> None:
   if [ ${#MATCH[@]} = 0 ]; then
     if [ "$COMP_TYPE" = 63 ]; then
       printf "%(b)s\\n%%s%%s\\n%(n)s" "$MESSAGE" "$MATCHED" >&2
-      kill -WINCH $$
+      printf '\\033[5n' >&2
     fi
   else
     for match in "${MATCH[@]}"; do

jsonargparse/_core.py

@@ -887,16 +887,24 @@ def _dump_delete_default_entries(self, subcfg, subdefaults):
                 val = subcfg[key]
                 default = subdefaults[key]
                 class_object_val = None
+                same_class_path = True
                 if is_subclass_spec(val):
+                    if not isinstance(default, dict):
+                        default = {}
                     if val["class_path"] != default.get("class_path"):
+                        same_class_path = False
                         with parser_context(parent_parser=self):
                             parser = ActionTypeHint.get_class_parser(val["class_path"])
                         default = {"init_args": parser.get_defaults().as_dict()}
                     class_object_val = val
                     val = val.get("init_args")
                     default = default.get("init_args")
                 if val == default:
-                    del subcfg[key]
+                    if same_class_path:
+                        del subcfg[key]
+                    else:
+                        # only the init_args are defaults, so the class_path is kept
+                        class_object_val.pop("init_args", None)
                 elif isinstance(val, dict) and isinstance(default, dict):
                     self._dump_delete_default_entries(val, default)
                     if class_object_val and class_object_val.get("init_args") == {}:

jsonargparse/_parameter_resolvers.py

@@ -802,12 +802,14 @@ def get_parameters_args_and_kwargs(self) -> tuple[ParamList, ParamList]:
 
         params_list = []
         removed_params: set[str] = set()
+        pop_or_get_params: set[str] = set()
         kwargs_value = kwargs_name and values_to_find[kwargs_name]
         kwargs_value_dump = kwargs_value and ast.dump(kwargs_value)
         for node, source in [(v, s) for k, v, s in values_found if k == kwargs_name]:
             if isinstance(node, ast.Call):
                 if ast_is_kwargs_pop_or_get(node, kwargs_value_dump):
                     param = self.get_kwargs_pop_or_get_parameter(node, self.component, self.parent, self.doc_params)
+                    pop_or_get_params.add(param.name)
                     params_list.append([param])
                     continue
                 kwarg = ast_get_call_kwarg_with_value(node, kwargs_value)
@@ -840,6 +842,8 @@ def get_parameters_args_and_kwargs(self) -> tuple[ParamList, ParamList]:
                     self.log_debug(f"unsupported type of assign: {ast_str(node)}")
 
         params = group_parameters(params_list)
+        # a pop/get from kwargs means the parameter is accepted, even if the value is then given explicitly
+        removed_params -= pop_or_get_params
         params = [p for p in params if p.name not in removed_params]
         return split_args_and_kwargs(params)
 

jsonargparse/_typehints.py

@@ -2424,18 +2424,84 @@
     return action_or_typehint
 
 
+module_prefix_pattern = re.compile(r"(?<![\w.])(?:[A-Za-z_][A-Za-z0-9_]*\.)+")
+none_type_pattern = re.compile(r"\bNone(Type)?\b")
+type_arg_prefix = "jsonargparseTypeArg"
+type_arg_pattern = re.compile(rf"\b{type_arg_prefix}\d+\b")
+
+
 def strip_module_names(string: str) -> str:
-    return re.sub(r"[A-Za-z0-9_<>.]+\.", "", string)
+    return module_prefix_pattern.sub("", string)
 
 
 def type_to_str(obj):
     if obj is ModuleType:
         return "ModuleType"
     if obj in type_expression_types:
         return type_expression_types[obj]
-    if obj in {bool, tuple} or is_subclass(obj, (int, float, str, Path, Enum)):
+    # is_subclass not used, since in python<3.12 it considers an Annotated a subclass of its origin type
+    if obj in {bool, tuple} or (isinstance(obj, type) and issubclass(obj, (int, float, str, Path, Enum))):
         return obj.__name__
-    return strip_module_names(str(obj)).replace("NoneType", "null")
+    return typehint_to_str(obj)
+
+
+def typehint_to_str(typehint) -> str:
+    """Type hint as a string, recreating it with subtypes replaced by their string form.
+
+    Only the outermost level goes through ``strip_module_names``, such that literal
+    values and metadata, e.g. floats and dotted strings, are never mangled by it.
+    """
+    if get_typehint_origin(typehint) is Literal:
+        values = ", ".join(repr_to_str(v) for v in typehint.__args__)
+        return f"Literal[{values}]"
+    if hasattr(typehint, "__metadata__"):
+        subtypes = [subtypehint_to_str(typehint.__origin__)] + [repr(m) for m in typehint.__metadata__]
+        return f"Annotated[{', '.join(subtypes)}]"
+
+    args = getattr(typehint, "__args__", None)
+    if isinstance(args, tuple) and args:
+        arg_subtypes = {}
+        new_args = []
+        for num, arg in enumerate(args):
+            if arg is NoneType or arg is Ellipsis:
+                new_args.append(arg)
+                continue
+            name = f"{type_arg_prefix}{num}"
+            arg_subtypes[name] = subtypehint_to_str(arg)
+            new_args.append(type(name, (), {}))
+        shallow = replace_typehint_args(typehint, new_args)
+        if shallow is not None:
+            string = none_type_pattern.sub("null", strip_module_names(str(shallow)))
+            return type_arg_pattern.sub(lambda match: arg_subtypes[match.group()], string)
+
+    return none_type_pattern.sub("null", strip_module_names(str(typehint)))
+
+
+def subtypehint_to_str(typehint) -> str:
+    if isinstance(typehint, type) and not getattr(typehint, "__args__", None):
+        return typehint.__name__  # the str of a class has a <class ...> wrap and the module name
+    return typehint_to_str(typehint)
+
+
+def replace_typehint_args(typehint, args):
+    """Same type hint but with its subtypes replaced, or None if not possible."""
+    if isinstance(typehint, UnionType):
+        return reduce(or_, args)
+    if isinstance(typehint, GenericAlias):
+        origin = typehint.__origin__
+        if origin in callable_origin_types:
+            # the input types of a callable are given as a list, e.g. Callable[[int], str]
+            return origin[args[0] if args[0] is Ellipsis else list(args[:-1]), args[-1]]
+        return GenericAlias(origin, tuple(args))
+    try:
+        return typehint.copy_with(tuple(args))
+    except Exception:
+        # no copy_with, e.g. a parameterized type that is a class, or it rejects the given subtypes
+        return None
+
+
+def repr_to_str(val):
+    return "null" if val is None else repr(val)
 
 
 def literal_to_str(val):

jsonargparse_tests/test_parameter_resolvers.py

@@ -484,6 +484,35 @@ def function_pop_get_conditional(p1: str, **kw):  # pragma: no cover
         kw.get("p3", "y")
 
 
+class ClassPopParent:  # pragma: no cover
+    def __init__(self, pp1: Optional[list] = None, pp2: int = 0):
+        """
+        Args:
+            pp1: help for pp1
+            pp2: help for pp2
+        """
+
+
+class ClassPopForward(ClassPopParent):  # pragma: no cover
+    def __init__(self, pf1: int = 0, **kwargs):
+        """
+        Args:
+            pf1: help for pf1
+            pp1: help for pp1
+        """
+        pp1 = list(kwargs.pop("pp1", None) or [])
+        super().__init__(pp1=pp1, **kwargs)
+
+
+def function_pop_and_forward(**kwargs):  # pragma: no cover
+    """
+    Args:
+        k1: help for k1
+    """
+    k1 = kwargs.pop("k1", 3)
+    return function_with_kwargs(k1=k1, **kwargs)
+
+
 def function_with_bug(**kws):  # pragma: no cover
     return does_not_exist(**kws)  # noqa: F821
 
@@ -863,6 +892,20 @@ def test_get_params_function_pop_get_conditional():
     )
 
 
+def test_get_params_class_pop_from_kwargs_and_forward():
+    params = get_params(ClassPopForward)
+    assert_params(params, ["pf1", "pp1", "pp2"])
+    assert params[1].annotation is inspect._empty
+    assert params[1].default is None
+
+
+def test_get_params_function_pop_from_kwargs_and_forward():
+    params = get_params(function_pop_and_forward)
+    assert_params(params, ["k1", "pk1", "k2"])
+    assert params[0].annotation is inspect._empty
+    assert params[0].default == 3
+
+
 def test_get_params_function_module_class():
     params = get_params(function_module_class)
     assert ["firstweekday"] == [p.name for p in params]

jsonargparse_tests/test_postponed_annotations.py

@@ -388,7 +388,7 @@ def test_unresolvable_annotation_help(parser):
     if sys.version_info < (3, 14):
         optional = "Optional[Unvalidated<MisspelledType>]"
     else:
-        optional = "None | Unvalidated<MisspelledType>"
+        optional = "null | Unvalidated<MisspelledType>"
     assert f"--fn.typo TYPO (type: {optional}, default: null)" in help_str
 
 

jsonargparse_tests/test_shtab.py

@@ -1,8 +1,14 @@
 import dataclasses
+import os
 import re
+import select
 import shlex
+import struct
 import subprocess
+import sys
 import tempfile
+import time
+from contextlib import suppress
 from enum import Enum
 from importlib.util import find_spec
 from os import PathLike
@@ -87,6 +93,8 @@
                     assert f"Expected type: {typehint}; Accepted by subclasses: {extra}" in err.decode()
                 if is_positional(dest, parser):
                     assert f"Argument: {dest}; Expected type: {typehint}" in err.decode()
+                redraw_requested = "\x1b[5n" in err.decode()
+                assert redraw_requested == (choices == []), "device status report expected iff there are no completions"
 
 
 def test_bash_any(parser, subtests):
@@ -293,6 +301,77 @@
     )
 
 
+def test_bash_script_binds_redraw_current_line(parser):
+    parser.add_argument("--num", type=int)
+    shtab_script = get_shtab_script(parser, "bash")
+    assert "bind '\"\\e[0n\": redraw-current-line'" in shtab_script
+
+
+def get_bash_major_version():
+    out = subprocess.run(["bash", "-c", 'echo "${BASH_VERSINFO[0]}"'], capture_output=True)
+    try:
+        return int(out.stdout.strip())
+    except ValueError:  # pragma: no cover
+        return 0
+
+
+def read_from_pty_until(fd, pattern, timeout=10.0):
+    out = b""
+    end = time.monotonic() + timeout
+    while pattern not in out and time.monotonic() < end:
+        ready, _, _ = select.select([fd], [], [], 0.5)
+        if ready:
+            try:
+                data = os.read(fd, 65536)
+            except OSError:  # pragma: no cover
+                break
+            if not data:
+                break  # pragma: no cover
+            out += data
+    return out
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="pty is not available on Windows")
+@pytest.mark.filterwarnings("ignore:.*multi-threaded, use of forkpty.*:DeprecationWarning")
+def test_bash_interactive_no_completions_redraws_prompt(parser, tmp_path):
+    if get_bash_major_version() < 4:
+        pytest.skip("test requires bash>=4")  # pragma: no cover
+    import fcntl
+    import pty
+    import termios
+
+    parser.add_argument("--num", type=int)
+    shtab_script_path = tmp_path / "comp.sh"
+    shtab_script_path.write_text(get_shtab_script(parser, "bash"))
+    rcfile = tmp_path / "rcfile"
+    rcfile.write_text(f"PS1='PROMPT$ '\nsource {shtab_script_path}\n")
+
+    pid, fd = pty.fork()
+    if pid == 0:  # pragma: no cover
+        try:
+            os.environ["TERM"] = "xterm-256color"
+            os.execvp("bash", ["bash", "--noprofile", "--rcfile", str(rcfile), "-i"])
+        finally:
+            os._exit(1)
+    try:
+        fcntl.ioctl(fd, termios.TIOCSWINSZ, struct.pack("HHHH", 40, 200, 0, 0))
+        read_from_pty_until(fd, b"PROMPT$ ")
+        os.write(fd, b"tool --num \t\t")
+        out = read_from_pty_until(fd, b"\x1b[5n")
+        assert b"Expected type: int" in out
+        assert b"\x1b[5n" in out, "completion should request a device status report from the terminal"
+        os.write(fd, b"\x1b[0n")  # a real terminal replies this to the \x1b[5n device status report
+        out = read_from_pty_until(fd, b"PROMPT$ tool --num ")
+        assert b"PROMPT$ tool --num " in out, "prompt should be redrawn after the guidance message"
+    finally:
+        with suppress(OSError):
+            os.write(fd, b"\x03exit\n")
+        with suppress(OSError):
+            os.close(fd)
+        with suppress(OSError):
+            os.waitpid(pid, 0)
+
+
 def test_bash_config(parser):
     parser.add_argument("--cfg", action="config")
     shtab_script = get_shtab_script(parser, "bash")

jsonargparse_tests/test_subclasses.py

@@ -2215,6 +2215,25 @@ def test_subclass_print_config(parser):
     assert "Option 'invalid' is not accepted" in err
 
 
+class PrintConfigOptional:
+    def __init__(self, sub: Optional[BaseC] = None):
+        pass  # pragma: no cover
+
+
+def test_subclass_print_config_skip_default_optional(parser):
+    parser.add_argument("--config", action="config")
+    parser.add_class_arguments(PrintConfigOptional, "g", sub_configs=True)
+
+    out = get_parse_args_stdout(parser, [f"--g.sub={__name__}.BaseC", "--print_config=skip_default"])
+    assert json_or_yaml_load(out) == {"g": {"sub": {"class_path": f"{__name__}.BaseC"}}}
+
+    out = get_parse_args_stdout(parser, [f"--g.sub={__name__}.BaseC", "--g.sub.p=3", "--print_config=skip_default"])
+    assert json_or_yaml_load(out) == {"g": {"sub": {"class_path": f"{__name__}.BaseC", "init_args": {"p": 3}}}}
+
+    out = get_parse_args_stdout(parser, ["--print_config=skip_default"])
+    assert json_or_yaml_load(out) in ({}, None)
+
+
 class PrintConfigRequired:
     def __init__(self, arg1: float):
         pass  # pragma: no cover