From f90eb564dde87b5801b589b1f66e628b71530d83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20R=C3=B8nning?= Date: Wed, 6 May 2026 12:26:52 +0200 Subject: [PATCH 1/5] feat(deploy): add human-readable drift view for cdf deploy Add --diff human to print a ToolkitPanel summary and a side-by-side comparison of CDF API dumps vs build YAML for changed resources. Verbose unified diff is skipped when human diff is selected. --- cognite_toolkit/_cdf_tk/apps/_core_app.py | 10 ++ cognite_toolkit/_cdf_tk/commands/__init__.py | 3 +- .../_cdf_tk/commands/deploy_v2/command.py | 15 +- .../commands/deploy_v2/diff_display.py | 147 ++++++++++++++++++ .../test_deployv2/test_diff_display.py | 31 ++++ 5 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py create mode 100644 tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py diff --git a/cognite_toolkit/_cdf_tk/apps/_core_app.py b/cognite_toolkit/_cdf_tk/apps/_core_app.py index 623d666706..fb5acafb83 100644 --- a/cognite_toolkit/_cdf_tk/apps/_core_app.py +++ b/cognite_toolkit/_cdf_tk/apps/_core_app.py @@ -22,6 +22,7 @@ BuildV2Command, CleanCommand, DeployCommand, + DeployDiffFormat, DeployOptions, DeployV2Command, ) @@ -520,6 +521,14 @@ def deploy_v2( help="Turn on to get more verbose output when running the command", ), ] = False, + diff: Annotated[ + DeployDiffFormat | None, + typer.Option( + "--diff", + help="When set to 'human', show a ToolkitPanel summary and a side-by-side line comparison of " + "CDF (API) dump vs build YAML for each resource that would change.", + ), + ] = None, ) -> None: """Deploys the configuration files in the build directory to the CDF project.""" if drop: @@ -548,6 +557,7 @@ def deploy_v2( include=include, force_update=force_update, verbose=verbose, + diff=diff, environment_variables=env_vars.dump(), deployment_dir=deploy_dir, ), diff --git a/cognite_toolkit/_cdf_tk/commands/__init__.py b/cognite_toolkit/_cdf_tk/commands/__init__.py index b987fcdfa7..b2bb8c2864 100644 --- a/cognite_toolkit/_cdf_tk/commands/__init__.py +++ b/cognite_toolkit/_cdf_tk/commands/__init__.py @@ -13,7 +13,7 @@ from .build_v2.build_v2 import BuildV2Command from .clean import CleanCommand from .deploy import DeployCommand -from .deploy_v2.command import DeploymentStep, DeployOptions, DeployV2Command +from .deploy_v2.command import DeployDiffFormat, DeploymentStep, DeployOptions, DeployV2Command from .dump_resource import DumpResourceCommand from .entity_matching import EntityMatchingCommand from .functions import FunctionsCommand @@ -31,6 +31,7 @@ "BuildV2Command", "CleanCommand", "DeployCommand", + "DeployDiffFormat", "DeployOptions", "DeployV2Command", "DeploymentStep", diff --git a/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py b/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py index e2bfbfd994..eea7895451 100644 --- a/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py +++ b/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py @@ -29,6 +29,7 @@ validate_soft_delete_capacity, ) from cognite_toolkit._cdf_tk.commands.build_v2.data_classes import BuildLineage +from cognite_toolkit._cdf_tk.commands.deploy_v2.diff_display import DeployDiffFormat, render_deploy_human_diff from cognite_toolkit._cdf_tk.constants import HINT_LEAD_TEXT from cognite_toolkit._cdf_tk.data_classes._tracking_info import DeploymentTracking from cognite_toolkit._cdf_tk.dataio.selectors import RawTableSelector, SelectedTable @@ -88,6 +89,7 @@ class DeployOptions: drop_data: bool = False environment_variables: dict[str, str | None] | None = None deployment_dir: Path | None = None + diff: DeployDiffFormat | None = None @dataclass @@ -874,7 +876,18 @@ def _categorize_resources( else: resources.to_delete.append(identifier) resources.to_create.append(resource.request) - if options.verbose: + if options.diff == DeployDiffFormat.human: + console.print( + render_deploy_human_diff( + resource_name=crud.display_name, + identifier=identifier, + source_file=resource.source_files[0], + cdf_dict=cdf_dict, + yaml_dict=resource.raw_dict, + sensitive_strings=crud.sensitive_strings(resource.request), + ) + ) + elif options.verbose: diff_str = "\n".join(to_diff(cdf_dict, resource.raw_dict)) for sensitive in crud.sensitive_strings(resource.request): diff_str = diff_str.replace(sensitive, "********") diff --git a/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py b/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py new file mode 100644 index 0000000000..e83e9ff079 --- /dev/null +++ b/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py @@ -0,0 +1,147 @@ +"""Human-readable deploy drift views (YAML vs CDF API).""" + +from __future__ import annotations + +from collections.abc import Iterable, Iterator, Sequence +from difflib import SequenceMatcher +from enum import Enum +from pathlib import Path +from typing import Any + +from rich.console import Group +from rich.text import Text + +from cognite_toolkit._cdf_tk.ui import AuraColor, ToolkitPanel, ToolkitPanelSection, ToolkitTable +from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump + + +class DeployDiffFormat(str, Enum): + human = "human" + + +def _sanitize(text: str, sensitive_strings: Iterable[str]) -> str: + for sensitive in sensitive_strings: + text = text.replace(sensitive, "********") + return text + + +def _summarize_opcodes(opcodes: Sequence[tuple[str, int, int, int, int]]) -> tuple[int, int, int, int]: + """Returns counts of: delete-only lines, insert-only lines, replace blocks, equal lines (excluding collapsed).""" + delete_lines = 0 + insert_lines = 0 + replace_blocks = 0 + equal_lines = 0 + for tag, i1, i2, j1, j2 in opcodes: + if tag == "delete": + delete_lines += i2 - i1 + elif tag == "insert": + insert_lines += j2 - j1 + elif tag == "replace": + replace_blocks += 1 + elif tag == "equal": + equal_lines += i2 - i1 + return delete_lines, insert_lines, replace_blocks, equal_lines + + +def _side_by_side_rows( + cdf_lines: list[str], + yaml_lines: list[str], + *, + equal_context: int = 3, + equal_collapse_at: int = 12, +) -> Iterator[tuple[Text, Text]]: + """Yield (left_cell, right_cell) for side-by-side diff rows.""" + matcher = SequenceMatcher(None, cdf_lines, yaml_lines) + opcodes = matcher.get_opcodes() + + for tag, i1, i2, j1, j2 in opcodes: + if tag == "equal": + span = i2 - i1 + if span <= equal_collapse_at: + for offset in range(span): + line_l = cdf_lines[i1 + offset] + line_r = yaml_lines[j1 + offset] + yield (Text(line_l, style="dim"), Text(line_r, style="dim")) + else: + head = equal_context + tail = equal_context + for offset in range(head): + yield ( + Text(cdf_lines[i1 + offset], style="dim"), + Text(yaml_lines[j1 + offset], style="dim"), + ) + omitted = span - head - tail + yield ( + Text(f"... {omitted} unchanged line(s) ...", style="italic dim"), + Text(f"... {omitted} unchanged line(s) ...", style="italic dim"), + ) + for offset in range(span - tail, span): + yield ( + Text(cdf_lines[i1 + offset], style="dim"), + Text(yaml_lines[j1 + offset], style="dim"), + ) + elif tag == "delete": + for idx in range(i1, i2): + yield (Text(cdf_lines[idx], style="red"), Text("-", style="dim")) + elif tag == "insert": + for idx in range(j1, j2): + yield (Text("-", style="dim"), Text(yaml_lines[idx], style="green")) + elif tag == "replace": + left = cdf_lines[i1:i2] + right = yaml_lines[j1:j2] + for k in range(max(len(left), len(right))): + l_txt = left[k] if k < len(left) else "" + r_txt = right[k] if k < len(right) else "" + yield (Text(l_txt, style="yellow"), Text(r_txt, style="cyan")) + + +def _build_side_by_side_table(cdf_lines: list[str], yaml_lines: list[str]) -> ToolkitTable: + table = ToolkitTable("CDF (API)", "Build (YAML)", expand=True) + table.columns[0].overflow = "fold" + table.columns[1].overflow = "fold" + for left, right in _side_by_side_rows(cdf_lines, yaml_lines): + table.add_row(left, right) + return table + + +def render_deploy_human_diff( + *, + resource_name: str, + identifier: Any, + source_file: Path, + cdf_dict: dict[str, Any], + yaml_dict: dict[str, Any], + sensitive_strings: Iterable[str], +) -> ToolkitPanel: + sens = list(sensitive_strings) + cdf_yaml = _sanitize(yaml_safe_dump(cdf_dict, sort_keys=True), sens) + build_yaml = _sanitize(yaml_safe_dump(yaml_dict, sort_keys=True), sens) + cdf_lines = cdf_yaml.splitlines() + yaml_lines = build_yaml.splitlines() + + matcher = SequenceMatcher(None, cdf_lines, yaml_lines) + delete_lines, insert_lines, replace_blocks, equal_lines = _summarize_opcodes(matcher.get_opcodes()) + + summary_lines = [ + f"[bold]{resource_name}[/] - [cyan]{identifier!s}[/]", + f"Source file: [dim]{source_file.as_posix()}[/]", + f"Serialized YAML: [dim]{len(cdf_lines)}[/] line(s) from CDF vs [dim]{len(yaml_lines)}[/] line(s) from build", + f"[green]✓[/] {equal_lines} unchanged line(s) in aligned YAML view", + f"[yellow]↔[/] {replace_blocks} replaced region(s)", + f"[red]-[/] {delete_lines} line(s) present in CDF only", + f"[green]+[/] {insert_lines} line(s) present in build only", + ] + + sections = [ + ToolkitPanelSection(title="Summary", content=summary_lines), + ToolkitPanelSection( + title="Line-by-line (API vs build YAML)", content=[_build_side_by_side_table(cdf_lines, yaml_lines)] + ), + ] + + return ToolkitPanel( + Group(*sections), + title=f"[bold]Deploy drift[/] - {resource_name}: {identifier!s}", + border_style=AuraColor.AMBER.rich, + expand=False, + ) diff --git a/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py b/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py new file mode 100644 index 0000000000..9d8fe6ed74 --- /dev/null +++ b/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py @@ -0,0 +1,31 @@ +from pathlib import Path + +from rich.console import Console + +from cognite_toolkit._cdf_tk.commands.deploy_v2.diff_display import DeployDiffFormat, render_deploy_human_diff + + +def test_deploy_diff_format_enum_value() -> None: + assert DeployDiffFormat.human.value == "human" + + +def test_render_deploy_human_diff_includes_summary_and_columns() -> None: + panel = render_deploy_human_diff( + resource_name="Data Sets", + identifier="my-dataset", + source_file=Path("modules/foo/datasets.Dataset.yaml"), + cdf_dict={"name": "a", "token": "SECRET"}, + yaml_dict={"name": "a", "token": "SECRET-other"}, + sensitive_strings=["SECRET"], + ) + console = Console(record=True, width=120, legacy_windows=False, color_system=None) + console.print(panel) + text = console.export_text(clear=False) + + assert "Summary" in text + assert "CDF (API)" in text + assert "Build (YAML)" in text + assert "my-dataset" in text + assert "modules/foo/datasets.Dataset.yaml" in text + assert "SECRET" not in text + assert "********" in text From 9fea04f609c4666603fbabed7438c1331945dd04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20R=C3=B8nning?= Date: Wed, 6 May 2026 13:09:20 +0200 Subject: [PATCH 2/5] savepoint --- .../_cdf_tk/commands/deploy_v2/command.py | 1 + .../commands/deploy_v2/diff_display.py | 50 +++++++++++++------ cognite_toolkit/_cdf_tk/ui.py | 49 +++++++++++++++++- .../test_deployv2/test_diff_display.py | 14 ++++-- 4 files changed, 91 insertions(+), 23 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py b/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py index eea7895451..c41babac3c 100644 --- a/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py +++ b/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py @@ -885,6 +885,7 @@ def _categorize_resources( cdf_dict=cdf_dict, yaml_dict=resource.raw_dict, sensitive_strings=crud.sensitive_strings(resource.request), + cdf_project=crud.client.config.project, ) ) elif options.verbose: diff --git a/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py b/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py index e83e9ff079..5d5a03737e 100644 --- a/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py +++ b/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py @@ -19,6 +19,10 @@ class DeployDiffFormat(str, Enum): human = "human" +# Rich Padding (top, right, bottom, left): one consistent body inset for every diff block. +_SECTION_BODY_PADDING = (0, 0, 1, 2) + + def _sanitize(text: str, sensitive_strings: Iterable[str]) -> str: for sensitive in sensitive_strings: text = text.replace(sensitive, "********") @@ -50,7 +54,7 @@ def _side_by_side_rows( equal_context: int = 3, equal_collapse_at: int = 12, ) -> Iterator[tuple[Text, Text]]: - """Yield (left_cell, right_cell) for side-by-side diff rows.""" + """Yield (CDF column cell, local build column cell); the table swaps them to show build left.""" matcher = SequenceMatcher(None, cdf_lines, yaml_lines) opcodes = matcher.get_opcodes() @@ -95,12 +99,15 @@ def _side_by_side_rows( yield (Text(l_txt, style="yellow"), Text(r_txt, style="cyan")) -def _build_side_by_side_table(cdf_lines: list[str], yaml_lines: list[str]) -> ToolkitTable: - table = ToolkitTable("CDF (API)", "Build (YAML)", expand=True) +def _build_side_by_side_table(cdf_lines: list[str], yaml_lines: list[str], *, cdf_project: str) -> ToolkitTable: + # Local build left, CDF (project) right — row cells are swapped from matcher order (CDF, build). + table = ToolkitTable("Local build", f"CDF ({cdf_project})", expand=True, padding=(0, 0)) table.columns[0].overflow = "fold" table.columns[1].overflow = "fold" - for left, right in _side_by_side_rows(cdf_lines, yaml_lines): - table.add_row(left, right) + table.columns[0].justify = "left" + table.columns[1].justify = "left" + for cdf_cell, build_cell in _side_by_side_rows(cdf_lines, yaml_lines): + table.add_row(build_cell, cdf_cell) return table @@ -112,6 +119,7 @@ def render_deploy_human_diff( cdf_dict: dict[str, Any], yaml_dict: dict[str, Any], sensitive_strings: Iterable[str], + cdf_project: str, ) -> ToolkitPanel: sens = list(sensitive_strings) cdf_yaml = _sanitize(yaml_safe_dump(cdf_dict, sort_keys=True), sens) @@ -123,25 +131,35 @@ def render_deploy_human_diff( delete_lines, insert_lines, replace_blocks, equal_lines = _summarize_opcodes(matcher.get_opcodes()) summary_lines = [ - f"[bold]{resource_name}[/] - [cyan]{identifier!s}[/]", - f"Source file: [dim]{source_file.as_posix()}[/]", - f"Serialized YAML: [dim]{len(cdf_lines)}[/] line(s) from CDF vs [dim]{len(yaml_lines)}[/] line(s) from build", - f"[green]✓[/] {equal_lines} unchanged line(s) in aligned YAML view", - f"[yellow]↔[/] {replace_blocks} replaced region(s)", - f"[red]-[/] {delete_lines} line(s) present in CDF only", - f"[green]+[/] {insert_lines} line(s) present in build only", + f"Serialized YAML: {len(cdf_lines)} line(s) from CDF vs {len(yaml_lines)} line(s) from build", + f"{equal_lines} unchanged line(s)", + f"{replace_blocks} replaced region(s)", + f"{delete_lines} line(s) present in {cdf_project} only", + f"{insert_lines} line(s) present in build only", ] sections = [ - ToolkitPanelSection(title="Summary", content=summary_lines), ToolkitPanelSection( - title="Line-by-line (API vs build YAML)", content=[_build_side_by_side_table(cdf_lines, yaml_lines)] + title="Resource", + content=[ + f"Type: {resource_name}", + f"Identifier: {identifier!s}", + f"Source file: {source_file.as_posix()}", + ], + content_padding=_SECTION_BODY_PADDING, + ), + ToolkitPanelSection( + title="Summary", + content=summary_lines, + content_padding=_SECTION_BODY_PADDING, + ), + ToolkitPanelSection( + content=[_build_side_by_side_table(cdf_lines, yaml_lines, cdf_project=cdf_project)], ), ] return ToolkitPanel( Group(*sections), - title=f"[bold]Deploy drift[/] - {resource_name}: {identifier!s}", + title=f"[bold]Diff view[/] - {resource_name}: {identifier!s}", border_style=AuraColor.AMBER.rich, - expand=False, ) diff --git a/cognite_toolkit/_cdf_tk/ui.py b/cognite_toolkit/_cdf_tk/ui.py index 81b929f3b4..8513c1dabd 100644 --- a/cognite_toolkit/_cdf_tk/ui.py +++ b/cognite_toolkit/_cdf_tk/ui.py @@ -43,6 +43,22 @@ def rich(self) -> str: class ToolkitPanel(Panel): + """Branded Rich :class:`~rich.panel.Panel` for toolkit CLI output (leading newline, default rounding). + + Constructor arguments are forwarded to :class:`~rich.panel.Panel` after normalizing string + ``title`` to bold :class:`~rich.text.Text`. + + The ``padding`` argument controls empty space **inside** the panel border, around the main + ``renderable``. It uses the same rules as Rich :class:`~rich.padding.Padding`: + + - **int** — same padding on all four sides. + - **pair** ``(vertical, horizontal)`` — top/bottom vs left/right. + - **4-tuple** ``(top, right, bottom, left)`` — explicit per side, in clockwise order from the top. + + Any remaining keyword arguments are passed through to :class:`~rich.panel.Panel` (e.g. + ``border_style``, ``expand``). + """ + def __init__( self, renderable: RenderableType, @@ -72,6 +88,28 @@ def __rich_console__(self, console: Console, options: ConsoleOptions) -> RenderR class ToolkitPanelSection(Group): + """A titled group of renderables, typically nested inside a :class:`ToolkitPanel`. + + Renders an optional bold ``title`` line (with trailing colon), optional ``description`` on + the same line, then the ``content`` items. Nested :class:`ToolkitPanelSection` instances are + indented with a fixed inset (:attr:`_nested_padding`). + + Args: + title: Optional section heading. When set, shown as ``"{title}:"`` in bold markup. + description: Optional text joined to the header after ``title``. + content: Renderables listed under the header (strings are rendered as markup). + content_padding: Optional extra space around the **body** only (everything under the + header). The header line is never padded. Uses Rich :class:`~rich.padding.Padding` + dimensions: + + - **int** — pad all sides by that many cells. + - **pair** ``(vertical, horizontal)`` — top/bottom vs left/right. + - **4-tuple** ``(top, right, bottom, left)`` — explicit per side (clockwise from top). + + For example, ``(0, 0, 0, 1)`` is one cell of **left** inset; ``(0, 0, 1, 0)`` is one row + of **bottom** spacing (third value is bottom, fourth is left). + """ + _nested_padding: ClassVar[PaddingDimensions] = (0, 0, 1, 2) def __init__( @@ -79,6 +117,7 @@ def __init__( title: str | Text | None = None, description: str | Text | None = None, content: Sequence[RenderableType] | None = None, + content_padding: PaddingDimensions | None = None, ) -> None: renderables: list[RenderableType] = [] header = f"[bold]{title}:[/]" if title else "" @@ -87,11 +126,17 @@ def __init__( if header: renderables.append(header) + body_items: list[RenderableType] = [] for item in content or []: if isinstance(item, ToolkitPanelSection): - renderables.append(Padding(item, self._nested_padding)) + body_items.append(Padding(item, self._nested_padding)) else: - renderables.append(item) + body_items.append(item) + if body_items: + body: RenderableType = body_items[0] if len(body_items) == 1 else Group(*body_items) + if content_padding is not None: + body = Padding(body, content_padding) + renderables.append(body) super().__init__(*renderables) diff --git a/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py b/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py index 9d8fe6ed74..ca774a82d0 100644 --- a/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py +++ b/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py @@ -1,3 +1,4 @@ +from io import StringIO from pathlib import Path from rich.console import Console @@ -17,14 +18,17 @@ def test_render_deploy_human_diff_includes_summary_and_columns() -> None: cdf_dict={"name": "a", "token": "SECRET"}, yaml_dict={"name": "a", "token": "SECRET-other"}, sensitive_strings=["SECRET"], + cdf_project="my-cdf-project", ) - console = Console(record=True, width=120, legacy_windows=False, color_system=None) + buf = StringIO() + console = Console(file=buf, width=200, legacy_windows=False, color_system=None) console.print(panel) - text = console.export_text(clear=False) + text = buf.getvalue() - assert "Summary" in text - assert "CDF (API)" in text - assert "Build (YAML)" in text + assert "Summary:" in text + assert "Diff line by line" not in text + assert "CDF (my-cdf-project)" in text + assert "Local build" in text assert "my-dataset" in text assert "modules/foo/datasets.Dataset.yaml" in text assert "SECRET" not in text From bf21f79ad59c5deb173f9dd8c851dcb2f78b908c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20R=C3=B8nning?= Date: Wed, 6 May 2026 13:26:24 +0200 Subject: [PATCH 3/5] savepoint --- .../_cdf_tk/commands/deploy_v2/command.py | 7 ++- .../commands/deploy_v2/diff_display.py | 44 ++++++++++++++++--- .../test_deployv2/test_command.py | 19 ++++++++ .../test_deployv2/test_diff_display.py | 24 +++++++++- 4 files changed, 85 insertions(+), 9 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py b/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py index c41babac3c..9334820643 100644 --- a/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py +++ b/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py @@ -1114,7 +1114,12 @@ def _display_results( skipped=[], is_missing_write_acl=False, ) - for result in results: + display_results = sorted( + results, + key=lambda r: (r.created_count, r.updated_count, r.deleted_count, r.unchanged_count), + reverse=True, + ) + for result in display_results: row = [ result.resource_name, str(result.created_count), diff --git a/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py b/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py index 5d5a03737e..9f187b00a4 100644 --- a/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py +++ b/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py @@ -23,6 +23,35 @@ class DeployDiffFormat(str, Enum): _SECTION_BODY_PADDING = (0, 0, 1, 2) +def _align_nested_dict_pair_for_yaml(cdf: Any, build: Any) -> tuple[Any, Any]: + """Return copies of ``cdf`` / ``build`` with identical dict key order for paired dumps. + + Order at each dict level: keys from **build** (insertion order), then keys only in **CDF** + (insertion order). Nested dict values are aligned the same way so ``yaml_safe_dump(..., + sort_keys=False)`` produces line-aligned text without global alphabetical sorting. + + Non-dict values and mismatched types are left unchanged; lists are not reordered. + """ + if isinstance(cdf, dict) and isinstance(build, dict): + keys = list(dict.fromkeys(list(build.keys()) + [k for k in cdf if k not in build])) + out_cdf: dict[str, Any] = {} + out_build: dict[str, Any] = {} + for k in keys: + cv = cdf.get(k) + bv = build.get(k) + if isinstance(cv, dict) and isinstance(bv, dict): + out_cdf[k], out_build[k] = _align_nested_dict_pair_for_yaml(cv, bv) + elif isinstance(cv, dict): + out_cdf[k], out_build[k] = _align_nested_dict_pair_for_yaml(cv, {}) + elif isinstance(bv, dict): + out_cdf[k], out_build[k] = _align_nested_dict_pair_for_yaml({}, bv) + else: + out_cdf[k] = cv + out_build[k] = bv + return out_cdf, out_build + return cdf, build + + def _sanitize(text: str, sensitive_strings: Iterable[str]) -> str: for sensitive in sensitive_strings: text = text.replace(sensitive, "********") @@ -122,8 +151,9 @@ def render_deploy_human_diff( cdf_project: str, ) -> ToolkitPanel: sens = list(sensitive_strings) - cdf_yaml = _sanitize(yaml_safe_dump(cdf_dict, sort_keys=True), sens) - build_yaml = _sanitize(yaml_safe_dump(yaml_dict, sort_keys=True), sens) + cdf_aligned, build_aligned = _align_nested_dict_pair_for_yaml(cdf_dict, yaml_dict) + cdf_yaml = _sanitize(yaml_safe_dump(cdf_aligned, sort_keys=False, indent=2), sens) + build_yaml = _sanitize(yaml_safe_dump(build_aligned, sort_keys=False, indent=2), sens) cdf_lines = cdf_yaml.splitlines() yaml_lines = build_yaml.splitlines() @@ -131,11 +161,11 @@ def render_deploy_human_diff( delete_lines, insert_lines, replace_blocks, equal_lines = _summarize_opcodes(matcher.get_opcodes()) summary_lines = [ - f"Serialized YAML: {len(cdf_lines)} line(s) from CDF vs {len(yaml_lines)} line(s) from build", - f"{equal_lines} unchanged line(s)", - f"{replace_blocks} replaced region(s)", - f"{delete_lines} line(s) present in {cdf_project} only", - f"{insert_lines} line(s) present in build only", + f"Serialized YAML: [dim]{len(cdf_lines)}[/] line(s) from CDF vs [dim]{len(yaml_lines)}[/] line(s) from build", + f"[green]✓[/] [bold]{equal_lines}[/] unchanged line(s)", + f"[yellow]↔[/] [bold]{replace_blocks}[/] replaced region(s)", + f"[red]-[/] [bold]{delete_lines}[/] line(s) only in CDF ([cyan]{cdf_project}[/])", + f"[green]+[/] [bold]{insert_lines}[/] line(s) only in local build", ] sections = [ diff --git a/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_command.py b/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_command.py index 1d351cf4a2..58cdde021b 100644 --- a/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_command.py +++ b/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_command.py @@ -609,3 +609,22 @@ def test_deploy_resources_raises_resource_creation_error_on_unexpected_api_respo "type": "missing", } ] + + +class TestDisplayDeployResultsOrder: + def test_deploy_summary_table_sorted_descending_create_update_delete_unchanged(self) -> None: + """Rows follow descending (created, updated, deleted, unchanged) so the biggest changes float up.""" + from io import StringIO + + from rich.console import Console + + buf = StringIO() + console = Console(file=buf, width=160, color_system=None, legacy_windows=False) + results = [ + DeploymentResult("zzz", True, 0, 0, 0, 5, False, []), + DeploymentResult("aaa", True, 3, 0, 0, 0, False, []), + DeploymentResult("bbb", True, 3, 0, 1, 0, False, []), + ] + DeployV2Command._display_results(results, "deploy", console, verbose=False) + text = buf.getvalue() + assert text.index("bbb") < text.index("aaa") < text.index("zzz") diff --git a/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py b/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py index ca774a82d0..791dccd9f1 100644 --- a/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py +++ b/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py @@ -3,13 +3,35 @@ from rich.console import Console -from cognite_toolkit._cdf_tk.commands.deploy_v2.diff_display import DeployDiffFormat, render_deploy_human_diff +from cognite_toolkit._cdf_tk.commands.deploy_v2.diff_display import ( + DeployDiffFormat, + _align_nested_dict_pair_for_yaml, + render_deploy_human_diff, +) def test_deploy_diff_format_enum_value() -> None: assert DeployDiffFormat.human.value == "human" +def test_align_nested_dict_pair_preserves_build_key_order_then_cdf_only() -> None: + cdf = {"z": 1, "a": 2, "only_cdf": 0} + build = {"a": 2, "z": 9} + c2, b2 = _align_nested_dict_pair_for_yaml(cdf, build) + assert list(c2.keys()) == list(b2.keys()) + assert list(c2.keys()) == ["a", "z", "only_cdf"] + assert c2["only_cdf"] == 0 + assert b2["only_cdf"] is None + + +def test_align_nested_dict_pair_aligns_nested_dict_key_order() -> None: + cdf = {"top": {"m": 1, "n": 2}} + build = {"top": {"n": 2, "m": 1}} + c2, b2 = _align_nested_dict_pair_for_yaml(cdf, build) + assert list(c2["top"].keys()) == list(b2["top"].keys()) + assert list(c2["top"].keys()) == ["n", "m"] + + def test_render_deploy_human_diff_includes_summary_and_columns() -> None: panel = render_deploy_human_diff( resource_name="Data Sets", From e5b9a4d1b0492571597241622369eaacd756a774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20R=C3=B8nning?= Date: Wed, 6 May 2026 13:31:42 +0200 Subject: [PATCH 4/5] fix(deploy): avoid coercing scalars to {} in YAML alignment Gemini review: only recurse with an empty dict when the key is missing on the other side, not when types disagree (dict vs scalar). Adds regression tests for dict/scalar mismatches in both directions. --- .../_cdf_tk/commands/deploy_v2/diff_display.py | 6 ++++-- .../test_deployv2/test_diff_display.py | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py b/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py index 9f187b00a4..9953e1fb83 100644 --- a/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py +++ b/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py @@ -31,6 +31,8 @@ def _align_nested_dict_pair_for_yaml(cdf: Any, build: Any) -> tuple[Any, Any]: sort_keys=False)`` produces line-aligned text without global alphabetical sorting. Non-dict values and mismatched types are left unchanged; lists are not reordered. + Empty ``{}`` is used only when a dict exists on one side and the **key** is absent on the other, + so scalar-vs-dict mismatches are not coerced to ``{}``. """ if isinstance(cdf, dict) and isinstance(build, dict): keys = list(dict.fromkeys(list(build.keys()) + [k for k in cdf if k not in build])) @@ -41,9 +43,9 @@ def _align_nested_dict_pair_for_yaml(cdf: Any, build: Any) -> tuple[Any, Any]: bv = build.get(k) if isinstance(cv, dict) and isinstance(bv, dict): out_cdf[k], out_build[k] = _align_nested_dict_pair_for_yaml(cv, bv) - elif isinstance(cv, dict): + elif isinstance(cv, dict) and k not in build: out_cdf[k], out_build[k] = _align_nested_dict_pair_for_yaml(cv, {}) - elif isinstance(bv, dict): + elif isinstance(bv, dict) and k not in cdf: out_cdf[k], out_build[k] = _align_nested_dict_pair_for_yaml({}, bv) else: out_cdf[k] = cv diff --git a/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py b/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py index 791dccd9f1..f05c578b8f 100644 --- a/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py +++ b/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py @@ -32,6 +32,22 @@ def test_align_nested_dict_pair_aligns_nested_dict_key_order() -> None: assert list(c2["top"].keys()) == ["n", "m"] +def test_align_nested_dict_pair_dict_vs_scalar_keeps_values() -> None: + cdf = {"x": {"nested": 1}} + build = {"x": "scalar"} + c2, b2 = _align_nested_dict_pair_for_yaml(cdf, build) + assert c2["x"] == {"nested": 1} + assert b2["x"] == "scalar" + + +def test_align_nested_dict_pair_build_dict_cdf_scalar_keeps_values() -> None: + cdf = {"x": "scalar"} + build = {"x": {"nested": 1}} + c2, b2 = _align_nested_dict_pair_for_yaml(cdf, build) + assert c2["x"] == "scalar" + assert b2["x"] == {"nested": 1} + + def test_render_deploy_human_diff_includes_summary_and_columns() -> None: panel = render_deploy_human_diff( resource_name="Data Sets", From 2a420676ce1f816af4851d38aa6a138e1e09592a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A5l=20R=C3=B8nning?= Date: Wed, 6 May 2026 16:26:15 +0200 Subject: [PATCH 5/5] fix(deploy): align read-dir panel and clarify hosted extractor warnings Flatten read-dir summary into the panel section list so bullets line up with the Plan block. Extract hosted extractor source/destination messages as constants and explain id-only CDF comparison. Drop unused future annotations from diff_display per project conventions. --- .../_cdf_tk/commands/deploy_v2/command.py | 2 +- .../commands/deploy_v2/diff_display.py | 2 -- .../_resource_ios/hosted_extractors.py | 19 +++++++++++++------ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py b/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py index 9334820643..cca664668f 100644 --- a/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py +++ b/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py @@ -381,7 +381,7 @@ def _display_setup( if invalid_yaml_file_count: read_dir_summary.append(f"[red]✗[/] [bold]{invalid_yaml_file_count}[/] invalid yaml files") - read_dir_subsections: list[RenderableType] = [ToolkitPanelSection(content=read_dir_summary)] + read_dir_subsections: list[RenderableType] = [*read_dir_summary] if verbose: if build_dir.skipped_directories: read_dir_subsections.append( diff --git a/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py b/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py index 9953e1fb83..23b36d28a0 100644 --- a/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py +++ b/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py @@ -1,7 +1,5 @@ """Human-readable deploy drift views (YAML vs CDF API).""" -from __future__ import annotations - from collections.abc import Iterable, Iterator, Sequence from difflib import SequenceMatcher from enum import Enum diff --git a/cognite_toolkit/_cdf_tk/resource_ios/_resource_ios/hosted_extractors.py b/cognite_toolkit/_cdf_tk/resource_ios/_resource_ios/hosted_extractors.py index fe5588656d..e618e6f578 100644 --- a/cognite_toolkit/_cdf_tk/resource_ios/_resource_ios/hosted_extractors.py +++ b/cognite_toolkit/_cdf_tk/resource_ios/_resource_ios/hosted_extractors.py @@ -52,6 +52,17 @@ from .data_organization import DataSetsIO +HOSTED_EXTRACTOR_SOURCE_COMPARE_NOTE = ( + "Hosted extractor sources are compared to CDF using only their external id (the full source " + "configuration is not read back from the API for diff). Deploy will therefore always include an update " + "for each source." +) +HOSTED_EXTRACTOR_DESTINATION_COMPARE_NOTE = ( + "Hosted extractor destinations are compared to CDF using only their external id (the full destination " + "configuration is not read back from the API for diff). Deploy will therefore always include an update " + "for each destination." +) + @final class HostedExtractorSourceIO( @@ -116,9 +127,7 @@ def _iterate( def dump_resource( self, resource: HostedExtractorSourceResponseUnion, local: dict[str, Any] | None = None ) -> dict[str, Any]: - HighSeverityWarning( - "Sources will always be considered different, and thus will always be redeployed." - ).print_warning(console=self.client.console) + HighSeverityWarning(HOSTED_EXTRACTOR_SOURCE_COMPARE_NOTE).print_warning(console=self.client.console) return self.dump_id(self.get_id(resource)) def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> HostedExtractorSourceRequestUnion: @@ -243,9 +252,7 @@ def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> H def dump_resource( self, resource: HostedExtractorDestinationResponse, local: dict[str, Any] | None = None ) -> dict[str, Any]: - HighSeverityWarning( - "Destinations will always be considered different, and thus will always be redeployed." - ).print_warning(console=self.client.console) + HighSeverityWarning(HOSTED_EXTRACTOR_DESTINATION_COMPARE_NOTE).print_warning(console=self.client.console) return self.dump_id(self.get_id(resource)) @classmethod