diff --git a/cognite_toolkit/_cdf_tk/apps/_core_app.py b/cognite_toolkit/_cdf_tk/apps/_core_app.py index 623d666706..fb5acafb83 100644 --- a/cognite_toolkit/_cdf_tk/apps/_core_app.py +++ b/cognite_toolkit/_cdf_tk/apps/_core_app.py @@ -22,6 +22,7 @@ BuildV2Command, CleanCommand, DeployCommand, + DeployDiffFormat, DeployOptions, DeployV2Command, ) @@ -520,6 +521,14 @@ def deploy_v2( help="Turn on to get more verbose output when running the command", ), ] = False, + diff: Annotated[ + DeployDiffFormat | None, + typer.Option( + "--diff", + help="When set to 'human', show a ToolkitPanel summary and a side-by-side line comparison of " + "CDF (API) dump vs build YAML for each resource that would change.", + ), + ] = None, ) -> None: """Deploys the configuration files in the build directory to the CDF project.""" if drop: @@ -548,6 +557,7 @@ def deploy_v2( include=include, force_update=force_update, verbose=verbose, + diff=diff, environment_variables=env_vars.dump(), deployment_dir=deploy_dir, ), diff --git a/cognite_toolkit/_cdf_tk/commands/__init__.py b/cognite_toolkit/_cdf_tk/commands/__init__.py index b987fcdfa7..b2bb8c2864 100644 --- a/cognite_toolkit/_cdf_tk/commands/__init__.py +++ b/cognite_toolkit/_cdf_tk/commands/__init__.py @@ -13,7 +13,7 @@ from .build_v2.build_v2 import BuildV2Command from .clean import CleanCommand from .deploy import DeployCommand -from .deploy_v2.command import DeploymentStep, DeployOptions, DeployV2Command +from .deploy_v2.command import DeployDiffFormat, DeploymentStep, DeployOptions, DeployV2Command from .dump_resource import DumpResourceCommand from .entity_matching import EntityMatchingCommand from .functions import FunctionsCommand @@ -31,6 +31,7 @@ "BuildV2Command", "CleanCommand", "DeployCommand", + "DeployDiffFormat", "DeployOptions", "DeployV2Command", "DeploymentStep", diff --git a/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py b/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py index e2bfbfd994..cca664668f 100644 --- a/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py +++ b/cognite_toolkit/_cdf_tk/commands/deploy_v2/command.py @@ -29,6 +29,7 @@ validate_soft_delete_capacity, ) from cognite_toolkit._cdf_tk.commands.build_v2.data_classes import BuildLineage +from cognite_toolkit._cdf_tk.commands.deploy_v2.diff_display import DeployDiffFormat, render_deploy_human_diff from cognite_toolkit._cdf_tk.constants import HINT_LEAD_TEXT from cognite_toolkit._cdf_tk.data_classes._tracking_info import DeploymentTracking from cognite_toolkit._cdf_tk.dataio.selectors import RawTableSelector, SelectedTable @@ -88,6 +89,7 @@ class DeployOptions: drop_data: bool = False environment_variables: dict[str, str | None] | None = None deployment_dir: Path | None = None + diff: DeployDiffFormat | None = None @dataclass @@ -379,7 +381,7 @@ def _display_setup( if invalid_yaml_file_count: read_dir_summary.append(f"[red]✗[/] [bold]{invalid_yaml_file_count}[/] invalid yaml files") - read_dir_subsections: list[RenderableType] = [ToolkitPanelSection(content=read_dir_summary)] + read_dir_subsections: list[RenderableType] = [*read_dir_summary] if verbose: if build_dir.skipped_directories: read_dir_subsections.append( @@ -874,7 +876,19 @@ def _categorize_resources( else: resources.to_delete.append(identifier) resources.to_create.append(resource.request) - if options.verbose: + if options.diff == DeployDiffFormat.human: + console.print( + render_deploy_human_diff( + resource_name=crud.display_name, + identifier=identifier, + source_file=resource.source_files[0], + cdf_dict=cdf_dict, + yaml_dict=resource.raw_dict, + sensitive_strings=crud.sensitive_strings(resource.request), + cdf_project=crud.client.config.project, + ) + ) + elif options.verbose: diff_str = "\n".join(to_diff(cdf_dict, resource.raw_dict)) for sensitive in crud.sensitive_strings(resource.request): diff_str = diff_str.replace(sensitive, "********") @@ -1100,7 +1114,12 @@ def _display_results( skipped=[], is_missing_write_acl=False, ) - for result in results: + display_results = sorted( + results, + key=lambda r: (r.created_count, r.updated_count, r.deleted_count, r.unchanged_count), + reverse=True, + ) + for result in display_results: row = [ result.resource_name, str(result.created_count), diff --git a/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py b/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py new file mode 100644 index 0000000000..23b36d28a0 --- /dev/null +++ b/cognite_toolkit/_cdf_tk/commands/deploy_v2/diff_display.py @@ -0,0 +1,195 @@ +"""Human-readable deploy drift views (YAML vs CDF API).""" + +from collections.abc import Iterable, Iterator, Sequence +from difflib import SequenceMatcher +from enum import Enum +from pathlib import Path +from typing import Any + +from rich.console import Group +from rich.text import Text + +from cognite_toolkit._cdf_tk.ui import AuraColor, ToolkitPanel, ToolkitPanelSection, ToolkitTable +from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump + + +class DeployDiffFormat(str, Enum): + human = "human" + + +# Rich Padding (top, right, bottom, left): one consistent body inset for every diff block. +_SECTION_BODY_PADDING = (0, 0, 1, 2) + + +def _align_nested_dict_pair_for_yaml(cdf: Any, build: Any) -> tuple[Any, Any]: + """Return copies of ``cdf`` / ``build`` with identical dict key order for paired dumps. + + Order at each dict level: keys from **build** (insertion order), then keys only in **CDF** + (insertion order). Nested dict values are aligned the same way so ``yaml_safe_dump(..., + sort_keys=False)`` produces line-aligned text without global alphabetical sorting. + + Non-dict values and mismatched types are left unchanged; lists are not reordered. + Empty ``{}`` is used only when a dict exists on one side and the **key** is absent on the other, + so scalar-vs-dict mismatches are not coerced to ``{}``. + """ + if isinstance(cdf, dict) and isinstance(build, dict): + keys = list(dict.fromkeys(list(build.keys()) + [k for k in cdf if k not in build])) + out_cdf: dict[str, Any] = {} + out_build: dict[str, Any] = {} + for k in keys: + cv = cdf.get(k) + bv = build.get(k) + if isinstance(cv, dict) and isinstance(bv, dict): + out_cdf[k], out_build[k] = _align_nested_dict_pair_for_yaml(cv, bv) + elif isinstance(cv, dict) and k not in build: + out_cdf[k], out_build[k] = _align_nested_dict_pair_for_yaml(cv, {}) + elif isinstance(bv, dict) and k not in cdf: + out_cdf[k], out_build[k] = _align_nested_dict_pair_for_yaml({}, bv) + else: + out_cdf[k] = cv + out_build[k] = bv + return out_cdf, out_build + return cdf, build + + +def _sanitize(text: str, sensitive_strings: Iterable[str]) -> str: + for sensitive in sensitive_strings: + text = text.replace(sensitive, "********") + return text + + +def _summarize_opcodes(opcodes: Sequence[tuple[str, int, int, int, int]]) -> tuple[int, int, int, int]: + """Returns counts of: delete-only lines, insert-only lines, replace blocks, equal lines (excluding collapsed).""" + delete_lines = 0 + insert_lines = 0 + replace_blocks = 0 + equal_lines = 0 + for tag, i1, i2, j1, j2 in opcodes: + if tag == "delete": + delete_lines += i2 - i1 + elif tag == "insert": + insert_lines += j2 - j1 + elif tag == "replace": + replace_blocks += 1 + elif tag == "equal": + equal_lines += i2 - i1 + return delete_lines, insert_lines, replace_blocks, equal_lines + + +def _side_by_side_rows( + cdf_lines: list[str], + yaml_lines: list[str], + *, + equal_context: int = 3, + equal_collapse_at: int = 12, +) -> Iterator[tuple[Text, Text]]: + """Yield (CDF column cell, local build column cell); the table swaps them to show build left.""" + matcher = SequenceMatcher(None, cdf_lines, yaml_lines) + opcodes = matcher.get_opcodes() + + for tag, i1, i2, j1, j2 in opcodes: + if tag == "equal": + span = i2 - i1 + if span <= equal_collapse_at: + for offset in range(span): + line_l = cdf_lines[i1 + offset] + line_r = yaml_lines[j1 + offset] + yield (Text(line_l, style="dim"), Text(line_r, style="dim")) + else: + head = equal_context + tail = equal_context + for offset in range(head): + yield ( + Text(cdf_lines[i1 + offset], style="dim"), + Text(yaml_lines[j1 + offset], style="dim"), + ) + omitted = span - head - tail + yield ( + Text(f"... {omitted} unchanged line(s) ...", style="italic dim"), + Text(f"... {omitted} unchanged line(s) ...", style="italic dim"), + ) + for offset in range(span - tail, span): + yield ( + Text(cdf_lines[i1 + offset], style="dim"), + Text(yaml_lines[j1 + offset], style="dim"), + ) + elif tag == "delete": + for idx in range(i1, i2): + yield (Text(cdf_lines[idx], style="red"), Text("-", style="dim")) + elif tag == "insert": + for idx in range(j1, j2): + yield (Text("-", style="dim"), Text(yaml_lines[idx], style="green")) + elif tag == "replace": + left = cdf_lines[i1:i2] + right = yaml_lines[j1:j2] + for k in range(max(len(left), len(right))): + l_txt = left[k] if k < len(left) else "" + r_txt = right[k] if k < len(right) else "" + yield (Text(l_txt, style="yellow"), Text(r_txt, style="cyan")) + + +def _build_side_by_side_table(cdf_lines: list[str], yaml_lines: list[str], *, cdf_project: str) -> ToolkitTable: + # Local build left, CDF (project) right — row cells are swapped from matcher order (CDF, build). + table = ToolkitTable("Local build", f"CDF ({cdf_project})", expand=True, padding=(0, 0)) + table.columns[0].overflow = "fold" + table.columns[1].overflow = "fold" + table.columns[0].justify = "left" + table.columns[1].justify = "left" + for cdf_cell, build_cell in _side_by_side_rows(cdf_lines, yaml_lines): + table.add_row(build_cell, cdf_cell) + return table + + +def render_deploy_human_diff( + *, + resource_name: str, + identifier: Any, + source_file: Path, + cdf_dict: dict[str, Any], + yaml_dict: dict[str, Any], + sensitive_strings: Iterable[str], + cdf_project: str, +) -> ToolkitPanel: + sens = list(sensitive_strings) + cdf_aligned, build_aligned = _align_nested_dict_pair_for_yaml(cdf_dict, yaml_dict) + cdf_yaml = _sanitize(yaml_safe_dump(cdf_aligned, sort_keys=False, indent=2), sens) + build_yaml = _sanitize(yaml_safe_dump(build_aligned, sort_keys=False, indent=2), sens) + cdf_lines = cdf_yaml.splitlines() + yaml_lines = build_yaml.splitlines() + + matcher = SequenceMatcher(None, cdf_lines, yaml_lines) + delete_lines, insert_lines, replace_blocks, equal_lines = _summarize_opcodes(matcher.get_opcodes()) + + summary_lines = [ + f"Serialized YAML: [dim]{len(cdf_lines)}[/] line(s) from CDF vs [dim]{len(yaml_lines)}[/] line(s) from build", + f"[green]✓[/] [bold]{equal_lines}[/] unchanged line(s)", + f"[yellow]↔[/] [bold]{replace_blocks}[/] replaced region(s)", + f"[red]-[/] [bold]{delete_lines}[/] line(s) only in CDF ([cyan]{cdf_project}[/])", + f"[green]+[/] [bold]{insert_lines}[/] line(s) only in local build", + ] + + sections = [ + ToolkitPanelSection( + title="Resource", + content=[ + f"Type: {resource_name}", + f"Identifier: {identifier!s}", + f"Source file: {source_file.as_posix()}", + ], + content_padding=_SECTION_BODY_PADDING, + ), + ToolkitPanelSection( + title="Summary", + content=summary_lines, + content_padding=_SECTION_BODY_PADDING, + ), + ToolkitPanelSection( + content=[_build_side_by_side_table(cdf_lines, yaml_lines, cdf_project=cdf_project)], + ), + ] + + return ToolkitPanel( + Group(*sections), + title=f"[bold]Diff view[/] - {resource_name}: {identifier!s}", + border_style=AuraColor.AMBER.rich, + ) diff --git a/cognite_toolkit/_cdf_tk/resource_ios/_resource_ios/hosted_extractors.py b/cognite_toolkit/_cdf_tk/resource_ios/_resource_ios/hosted_extractors.py index fe5588656d..e618e6f578 100644 --- a/cognite_toolkit/_cdf_tk/resource_ios/_resource_ios/hosted_extractors.py +++ b/cognite_toolkit/_cdf_tk/resource_ios/_resource_ios/hosted_extractors.py @@ -52,6 +52,17 @@ from .data_organization import DataSetsIO +HOSTED_EXTRACTOR_SOURCE_COMPARE_NOTE = ( + "Hosted extractor sources are compared to CDF using only their external id (the full source " + "configuration is not read back from the API for diff). Deploy will therefore always include an update " + "for each source." +) +HOSTED_EXTRACTOR_DESTINATION_COMPARE_NOTE = ( + "Hosted extractor destinations are compared to CDF using only their external id (the full destination " + "configuration is not read back from the API for diff). Deploy will therefore always include an update " + "for each destination." +) + @final class HostedExtractorSourceIO( @@ -116,9 +127,7 @@ def _iterate( def dump_resource( self, resource: HostedExtractorSourceResponseUnion, local: dict[str, Any] | None = None ) -> dict[str, Any]: - HighSeverityWarning( - "Sources will always be considered different, and thus will always be redeployed." - ).print_warning(console=self.client.console) + HighSeverityWarning(HOSTED_EXTRACTOR_SOURCE_COMPARE_NOTE).print_warning(console=self.client.console) return self.dump_id(self.get_id(resource)) def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> HostedExtractorSourceRequestUnion: @@ -243,9 +252,7 @@ def load_resource(self, resource: dict[str, Any], is_dry_run: bool = False) -> H def dump_resource( self, resource: HostedExtractorDestinationResponse, local: dict[str, Any] | None = None ) -> dict[str, Any]: - HighSeverityWarning( - "Destinations will always be considered different, and thus will always be redeployed." - ).print_warning(console=self.client.console) + HighSeverityWarning(HOSTED_EXTRACTOR_DESTINATION_COMPARE_NOTE).print_warning(console=self.client.console) return self.dump_id(self.get_id(resource)) @classmethod diff --git a/cognite_toolkit/_cdf_tk/ui.py b/cognite_toolkit/_cdf_tk/ui.py index 81b929f3b4..8513c1dabd 100644 --- a/cognite_toolkit/_cdf_tk/ui.py +++ b/cognite_toolkit/_cdf_tk/ui.py @@ -43,6 +43,22 @@ def rich(self) -> str: class ToolkitPanel(Panel): + """Branded Rich :class:`~rich.panel.Panel` for toolkit CLI output (leading newline, default rounding). + + Constructor arguments are forwarded to :class:`~rich.panel.Panel` after normalizing string + ``title`` to bold :class:`~rich.text.Text`. + + The ``padding`` argument controls empty space **inside** the panel border, around the main + ``renderable``. It uses the same rules as Rich :class:`~rich.padding.Padding`: + + - **int** — same padding on all four sides. + - **pair** ``(vertical, horizontal)`` — top/bottom vs left/right. + - **4-tuple** ``(top, right, bottom, left)`` — explicit per side, in clockwise order from the top. + + Any remaining keyword arguments are passed through to :class:`~rich.panel.Panel` (e.g. + ``border_style``, ``expand``). + """ + def __init__( self, renderable: RenderableType, @@ -72,6 +88,28 @@ def __rich_console__(self, console: Console, options: ConsoleOptions) -> RenderR class ToolkitPanelSection(Group): + """A titled group of renderables, typically nested inside a :class:`ToolkitPanel`. + + Renders an optional bold ``title`` line (with trailing colon), optional ``description`` on + the same line, then the ``content`` items. Nested :class:`ToolkitPanelSection` instances are + indented with a fixed inset (:attr:`_nested_padding`). + + Args: + title: Optional section heading. When set, shown as ``"{title}:"`` in bold markup. + description: Optional text joined to the header after ``title``. + content: Renderables listed under the header (strings are rendered as markup). + content_padding: Optional extra space around the **body** only (everything under the + header). The header line is never padded. Uses Rich :class:`~rich.padding.Padding` + dimensions: + + - **int** — pad all sides by that many cells. + - **pair** ``(vertical, horizontal)`` — top/bottom vs left/right. + - **4-tuple** ``(top, right, bottom, left)`` — explicit per side (clockwise from top). + + For example, ``(0, 0, 0, 1)`` is one cell of **left** inset; ``(0, 0, 1, 0)`` is one row + of **bottom** spacing (third value is bottom, fourth is left). + """ + _nested_padding: ClassVar[PaddingDimensions] = (0, 0, 1, 2) def __init__( @@ -79,6 +117,7 @@ def __init__( title: str | Text | None = None, description: str | Text | None = None, content: Sequence[RenderableType] | None = None, + content_padding: PaddingDimensions | None = None, ) -> None: renderables: list[RenderableType] = [] header = f"[bold]{title}:[/]" if title else "" @@ -87,11 +126,17 @@ def __init__( if header: renderables.append(header) + body_items: list[RenderableType] = [] for item in content or []: if isinstance(item, ToolkitPanelSection): - renderables.append(Padding(item, self._nested_padding)) + body_items.append(Padding(item, self._nested_padding)) else: - renderables.append(item) + body_items.append(item) + if body_items: + body: RenderableType = body_items[0] if len(body_items) == 1 else Group(*body_items) + if content_padding is not None: + body = Padding(body, content_padding) + renderables.append(body) super().__init__(*renderables) diff --git a/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_command.py b/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_command.py index 1d351cf4a2..58cdde021b 100644 --- a/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_command.py +++ b/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_command.py @@ -609,3 +609,22 @@ def test_deploy_resources_raises_resource_creation_error_on_unexpected_api_respo "type": "missing", } ] + + +class TestDisplayDeployResultsOrder: + def test_deploy_summary_table_sorted_descending_create_update_delete_unchanged(self) -> None: + """Rows follow descending (created, updated, deleted, unchanged) so the biggest changes float up.""" + from io import StringIO + + from rich.console import Console + + buf = StringIO() + console = Console(file=buf, width=160, color_system=None, legacy_windows=False) + results = [ + DeploymentResult("zzz", True, 0, 0, 0, 5, False, []), + DeploymentResult("aaa", True, 3, 0, 0, 0, False, []), + DeploymentResult("bbb", True, 3, 0, 1, 0, False, []), + ] + DeployV2Command._display_results(results, "deploy", console, verbose=False) + text = buf.getvalue() + assert text.index("bbb") < text.index("aaa") < text.index("zzz") diff --git a/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py b/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py new file mode 100644 index 0000000000..f05c578b8f --- /dev/null +++ b/tests/test_unit/test_cdf_tk/test_commands/test_deployv2/test_diff_display.py @@ -0,0 +1,73 @@ +from io import StringIO +from pathlib import Path + +from rich.console import Console + +from cognite_toolkit._cdf_tk.commands.deploy_v2.diff_display import ( + DeployDiffFormat, + _align_nested_dict_pair_for_yaml, + render_deploy_human_diff, +) + + +def test_deploy_diff_format_enum_value() -> None: + assert DeployDiffFormat.human.value == "human" + + +def test_align_nested_dict_pair_preserves_build_key_order_then_cdf_only() -> None: + cdf = {"z": 1, "a": 2, "only_cdf": 0} + build = {"a": 2, "z": 9} + c2, b2 = _align_nested_dict_pair_for_yaml(cdf, build) + assert list(c2.keys()) == list(b2.keys()) + assert list(c2.keys()) == ["a", "z", "only_cdf"] + assert c2["only_cdf"] == 0 + assert b2["only_cdf"] is None + + +def test_align_nested_dict_pair_aligns_nested_dict_key_order() -> None: + cdf = {"top": {"m": 1, "n": 2}} + build = {"top": {"n": 2, "m": 1}} + c2, b2 = _align_nested_dict_pair_for_yaml(cdf, build) + assert list(c2["top"].keys()) == list(b2["top"].keys()) + assert list(c2["top"].keys()) == ["n", "m"] + + +def test_align_nested_dict_pair_dict_vs_scalar_keeps_values() -> None: + cdf = {"x": {"nested": 1}} + build = {"x": "scalar"} + c2, b2 = _align_nested_dict_pair_for_yaml(cdf, build) + assert c2["x"] == {"nested": 1} + assert b2["x"] == "scalar" + + +def test_align_nested_dict_pair_build_dict_cdf_scalar_keeps_values() -> None: + cdf = {"x": "scalar"} + build = {"x": {"nested": 1}} + c2, b2 = _align_nested_dict_pair_for_yaml(cdf, build) + assert c2["x"] == "scalar" + assert b2["x"] == {"nested": 1} + + +def test_render_deploy_human_diff_includes_summary_and_columns() -> None: + panel = render_deploy_human_diff( + resource_name="Data Sets", + identifier="my-dataset", + source_file=Path("modules/foo/datasets.Dataset.yaml"), + cdf_dict={"name": "a", "token": "SECRET"}, + yaml_dict={"name": "a", "token": "SECRET-other"}, + sensitive_strings=["SECRET"], + cdf_project="my-cdf-project", + ) + buf = StringIO() + console = Console(file=buf, width=200, legacy_windows=False, color_system=None) + console.print(panel) + text = buf.getvalue() + + assert "Summary:" in text + assert "Diff line by line" not in text + assert "CDF (my-cdf-project)" in text + assert "Local build" in text + assert "my-dataset" in text + assert "modules/foo/datasets.Dataset.yaml" in text + assert "SECRET" not in text + assert "********" in text