From 35c4998852221830d7bdf431dbb53c8d174fa995 Mon Sep 17 00:00:00 2001 From: "Michiel De Smet (aider)" Date: Tue, 10 Jun 2025 13:33:13 +0800 Subject: [PATCH 1/8] refactor: move token and instance-name to top-level command --- src/datapilot/cli/main.py | 11 +++++- src/datapilot/core/platforms/dbt/cli/cli.py | 37 +++++++++++++-------- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/datapilot/cli/main.py b/src/datapilot/cli/main.py index f0f1796..cd8261a 100644 --- a/src/datapilot/cli/main.py +++ b/src/datapilot/cli/main.py @@ -5,8 +5,17 @@ @click.group() -def datapilot(): +@click.option("--token", required=False, help="Your API token for authentication.") +@click.option("--instance-name", required=False, help="Your tenant ID.") +@click.option("--backend-url", required=False, help="Altimate's Backend URL", default="https://api.myaltimate.com") +@click.pass_context +def datapilot(ctx, token, instance_name, backend_url): """Altimate CLI for DBT project management.""" + # Store common options in context + ctx.ensure_object(dict) + ctx.obj['token'] = token + ctx.obj['instance_name'] = instance_name + ctx.obj['backend_url'] = backend_url datapilot.add_command(dbt) diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 8472ebf..233f6b1 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -24,13 +24,14 @@ # New dbt group @click.group() -def dbt(): +@click.pass_context +def dbt(ctx): """DBT specific commands.""" + # Ensure context object exists + ctx.ensure_object(dict) @dbt.command("project-health") -@click.option("--token", required=False, help="Your API token for authentication.") -@click.option("--instance-name", required=False, help="Your tenant ID.") @click.option( "--manifest-path", required=True, @@ -57,21 +58,24 @@ def dbt(): default=None, help="Selective model testing. Specify one or more models to run tests on.", ) -@click.option("--backend-url", required=False, help="Altimate's Backend URL", default="https://api.myaltimate.com") +@click.pass_context def project_health( - token, - instance_name, + ctx, manifest_path, catalog_path, config_path=None, config_name=None, select=None, - backend_url="https://api.myaltimate.com", ): """ Validate the DBT project's configuration and structure. :param manifest_path: Path to the DBT manifest file. """ + # Get common options from parent context + token = ctx.parent.obj.get('token') + instance_name = ctx.parent.obj.get('instance_name') + backend_url = ctx.parent.obj.get('backend_url') + config = None if config_path: config = load_config(config_path) @@ -131,25 +135,32 @@ def project_health( @dbt.command("onboard") -@click.option("--token", prompt="API Token", help="Your API token for authentication.") -@click.option("--instance-name", prompt="Instance Name", help="Your tenant ID.") @click.option("--dbt_core_integration_id", prompt="DBT Core Integration ID", help="DBT Core Integration ID") @click.option( "--dbt_core_integration_environment", default="PROD", prompt="DBT Core Integration Environment", help="DBT Core Integration Environment" ) @click.option("--manifest-path", required=True, prompt="Manifest Path", help="Path to the manifest file.") @click.option("--catalog-path", required=False, prompt=False, help="Path to the catalog file.") -@click.option("--backend-url", required=False, help="Altimate's Backend URL", default="https://api.myaltimate.com") +@click.pass_context def onboard( - token, - instance_name, + ctx, dbt_core_integration_id, dbt_core_integration_environment, manifest_path, catalog_path, - backend_url="https://api.myaltimate.com", ): """Onboard a manifest file to DBT.""" + # Get common options from parent context + token = ctx.parent.obj.get('token') + instance_name = ctx.parent.obj.get('instance_name') + backend_url = ctx.parent.obj.get('backend_url') + + # For onboard command, token and instance_name are required + if not token: + token = click.prompt("API Token") + if not instance_name: + instance_name = click.prompt("Instance Name") + check_token_and_instance(token, instance_name) if not validate_credentials(token, backend_url, instance_name): From 98e123217359a0ddc857772f181e18108bab110c Mon Sep 17 00:00:00 2001 From: "Michiel De Smet (aider)" Date: Tue, 10 Jun 2025 13:37:41 +0800 Subject: [PATCH 2/8] feat: add config file support with env var substitution --- src/datapilot/cli/main.py | 82 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 4 deletions(-) diff --git a/src/datapilot/cli/main.py b/src/datapilot/cli/main.py index cd8261a..21bc2f1 100644 --- a/src/datapilot/cli/main.py +++ b/src/datapilot/cli/main.py @@ -1,9 +1,54 @@ +import json +import os +import re +from pathlib import Path + import click +from dotenv import load_dotenv from datapilot.core.mcp_utils.mcp import mcp from datapilot.core.platforms.dbt.cli.cli import dbt +def load_config_from_file(): + """Load configuration from ~/.altimate/altimate.json if it exists.""" + config_path = Path.home() / ".altimate" / "altimate.json" + + if not config_path.exists(): + return {} + + try: + with open(config_path, 'r') as f: + config = json.load(f) + return config + except (json.JSONDecodeError, IOError) as e: + click.echo(f"Warning: Failed to load config from {config_path}: {e}", err=True) + return {} + + +def substitute_env_vars(value): + """Replace ${env:ENV_VARIABLE} patterns with actual environment variable values.""" + if not isinstance(value, str): + return value + + # Pattern to match ${env:VARIABLE_NAME} + pattern = r'\$\{env:([^}]+)\}' + + def replacer(match): + env_var = match.group(1) + return os.environ.get(env_var, match.group(0)) + + return re.sub(pattern, replacer, value) + + +def process_config(config): + """Process configuration dictionary to substitute environment variables.""" + processed = {} + for key, value in config.items(): + processed[key] = substitute_env_vars(value) + return processed + + @click.group() @click.option("--token", required=False, help="Your API token for authentication.") @click.option("--instance-name", required=False, help="Your tenant ID.") @@ -11,11 +56,40 @@ @click.pass_context def datapilot(ctx, token, instance_name, backend_url): """Altimate CLI for DBT project management.""" - # Store common options in context + # Load .env file from current directory if it exists + load_dotenv() + + # Load configuration from file + file_config = load_config_from_file() + file_config = process_config(file_config) + + # Map config file keys to CLI option names + config_mapping = { + 'altimateApiKey': 'token', + 'altimateInstanceName': 'instance_name', + 'altimateUrl': 'backend_url' + } + + # Store common options in context, with CLI args taking precedence ctx.ensure_object(dict) - ctx.obj['token'] = token - ctx.obj['instance_name'] = instance_name - ctx.obj['backend_url'] = backend_url + + # Apply file config first + for file_key, cli_key in config_mapping.items(): + if file_key in file_config: + ctx.obj[cli_key] = file_config[file_key] + + # Override with CLI arguments if provided + if token is not None: + ctx.obj['token'] = token + if instance_name is not None: + ctx.obj['instance_name'] = instance_name + if backend_url != "https://api.myaltimate.com": # Only override if not default + ctx.obj['backend_url'] = backend_url + + # Set defaults if nothing was provided + ctx.obj.setdefault('token', None) + ctx.obj.setdefault('instance_name', None) + ctx.obj.setdefault('backend_url', 'https://api.myaltimate.com') datapilot.add_command(dbt) From e9d3f71500124c0d8e6be0574839c9d59e4cd593 Mon Sep 17 00:00:00 2001 From: "Michiel De Smet (aider)" Date: Tue, 10 Jun 2025 13:38:51 +0800 Subject: [PATCH 3/8] build: add python-dotenv dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 2da8546..d24279d 100644 --- a/setup.py +++ b/setup.py @@ -70,6 +70,7 @@ def read(*names, **kwargs): "sqlglot~=25.30.0", "mcp~=1.9.0", "pyperclip~=1.8.2", + "python-dotenv~=1.0.0", ], extras_require={ # eg: From ff3dca603a9788ecec3e0089623f49a22d6ee643 Mon Sep 17 00:00:00 2001 From: Michiel De Smet Date: Tue, 10 Jun 2025 13:53:32 +0800 Subject: [PATCH 4/8] refactor: standardize string quotes and improve code consistency --- src/datapilot/cli/main.py | 46 ++++++++++----------- src/datapilot/core/platforms/dbt/cli/cli.py | 18 ++++---- tests/core/platform/dbt/test_cli.py | 22 ++++++---- 3 files changed, 45 insertions(+), 41 deletions(-) diff --git a/src/datapilot/cli/main.py b/src/datapilot/cli/main.py index 21bc2f1..570050a 100644 --- a/src/datapilot/cli/main.py +++ b/src/datapilot/cli/main.py @@ -13,15 +13,15 @@ def load_config_from_file(): """Load configuration from ~/.altimate/altimate.json if it exists.""" config_path = Path.home() / ".altimate" / "altimate.json" - + if not config_path.exists(): return {} - + try: - with open(config_path, 'r') as f: + with open(config_path) as f: config = json.load(f) return config - except (json.JSONDecodeError, IOError) as e: + except (OSError, json.JSONDecodeError) as e: click.echo(f"Warning: Failed to load config from {config_path}: {e}", err=True) return {} @@ -30,14 +30,14 @@ def substitute_env_vars(value): """Replace ${env:ENV_VARIABLE} patterns with actual environment variable values.""" if not isinstance(value, str): return value - + # Pattern to match ${env:VARIABLE_NAME} - pattern = r'\$\{env:([^}]+)\}' - + pattern = r"\$\{env:([^}]+)\}" + def replacer(match): env_var = match.group(1) return os.environ.get(env_var, match.group(0)) - + return re.sub(pattern, replacer, value) @@ -58,38 +58,34 @@ def datapilot(ctx, token, instance_name, backend_url): """Altimate CLI for DBT project management.""" # Load .env file from current directory if it exists load_dotenv() - + # Load configuration from file file_config = load_config_from_file() file_config = process_config(file_config) - + # Map config file keys to CLI option names - config_mapping = { - 'altimateApiKey': 'token', - 'altimateInstanceName': 'instance_name', - 'altimateUrl': 'backend_url' - } - + config_mapping = {"altimateApiKey": "token", "altimateInstanceName": "instance_name", "altimateUrl": "backend_url"} + # Store common options in context, with CLI args taking precedence ctx.ensure_object(dict) - + # Apply file config first for file_key, cli_key in config_mapping.items(): if file_key in file_config: ctx.obj[cli_key] = file_config[file_key] - + # Override with CLI arguments if provided if token is not None: - ctx.obj['token'] = token + ctx.obj["token"] = token if instance_name is not None: - ctx.obj['instance_name'] = instance_name + ctx.obj["instance_name"] = instance_name if backend_url != "https://api.myaltimate.com": # Only override if not default - ctx.obj['backend_url'] = backend_url - + ctx.obj["backend_url"] = backend_url + # Set defaults if nothing was provided - ctx.obj.setdefault('token', None) - ctx.obj.setdefault('instance_name', None) - ctx.obj.setdefault('backend_url', 'https://api.myaltimate.com') + ctx.obj.setdefault("token", None) + ctx.obj.setdefault("instance_name", None) + ctx.obj.setdefault("backend_url", "https://api.myaltimate.com") datapilot.add_command(dbt) diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 233f6b1..0f63167 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -72,10 +72,10 @@ def project_health( :param manifest_path: Path to the DBT manifest file. """ # Get common options from parent context - token = ctx.parent.obj.get('token') - instance_name = ctx.parent.obj.get('instance_name') - backend_url = ctx.parent.obj.get('backend_url') - + token = ctx.parent.obj.get("token") + instance_name = ctx.parent.obj.get("instance_name") + backend_url = ctx.parent.obj.get("backend_url") + config = None if config_path: config = load_config(config_path) @@ -151,16 +151,16 @@ def onboard( ): """Onboard a manifest file to DBT.""" # Get common options from parent context - token = ctx.parent.obj.get('token') - instance_name = ctx.parent.obj.get('instance_name') - backend_url = ctx.parent.obj.get('backend_url') - + token = ctx.parent.obj.get("token") + instance_name = ctx.parent.obj.get("instance_name") + backend_url = ctx.parent.obj.get("backend_url") + # For onboard command, token and instance_name are required if not token: token = click.prompt("API Token") if not instance_name: instance_name = click.prompt("Instance Name") - + check_token_and_instance(token, instance_name) if not validate_credentials(token, backend_url, instance_name): diff --git a/tests/core/platform/dbt/test_cli.py b/tests/core/platform/dbt/test_cli.py index 04242b7..c79fc72 100644 --- a/tests/core/platform/dbt/test_cli.py +++ b/tests/core/platform/dbt/test_cli.py @@ -1,7 +1,7 @@ # test_app.py from click.testing import CliRunner -from datapilot.core.platforms.dbt.cli.cli import project_health +from datapilot.cli.main import datapilot def test_project_health_with_required_and_optional_args(): @@ -11,7 +11,9 @@ def test_project_health_with_required_and_optional_args(): config_path = "tests/data/config.yml" # Simulate command invocation - result = runner.invoke(project_health, ["--manifest-path", manifest_path, "--catalog-path", catalog_path, "--config-path", config_path]) + result = runner.invoke( + datapilot, ["project-health", "--manifest-path", manifest_path, "--catalog-path", catalog_path, "--config-path", config_path] + ) assert result.exit_code == 0 # Ensure the command executed successfully # Add more assertions here to validate the behavior of your command, @@ -25,8 +27,9 @@ def test_project_health_with_only_required_arg(): # Simulate command invocation without optional arguments result = runner.invoke( - project_health, + datapilot, [ + "project-health", "--manifest-path", manifest_path, ], @@ -43,8 +46,9 @@ def test_project_health_with_only_required_arg_version1_6(): # Simulate command invocation without optional arguments result = runner.invoke( - project_health, + datapilot, [ + "project-health", "--manifest-path", manifest_path, ], @@ -61,8 +65,9 @@ def test_project_health_with_macro_args(): # Simulate command invocation without optional arguments result = runner.invoke( - project_health, + datapilot, [ + "project-health", "--manifest-path", manifest_path, ], @@ -76,8 +81,9 @@ def test_project_health_with_macro_args(): # Simulate command invocation without optional arguments result = runner.invoke( - project_health, + datapilot, [ + "project-health", "--manifest-path", manifest_path, ], @@ -95,7 +101,9 @@ def test_project_health_with_required_and_optional_args_v12(): config_path = "tests/data/config.yml" # Simulate command invocation - result = runner.invoke(project_health, ["--manifest-path", manifest_path, "--catalog-path", catalog_path, "--config-path", config_path]) + result = runner.invoke( + datapilot, ["project-health", "--manifest-path", manifest_path, "--catalog-path", catalog_path, "--config-path", config_path] + ) assert result.exit_code == 0 # Ensure the command executed successfully # Add more assertions here to validate the behavior of your command, From 34288ef9e85e84d1813caecfc9741b760a808b18 Mon Sep 17 00:00:00 2001 From: Michiel De Smet Date: Tue, 10 Jun 2025 13:54:40 +0800 Subject: [PATCH 5/8] feat: hide token input in CLI for security --- src/datapilot/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datapilot/cli/main.py b/src/datapilot/cli/main.py index 570050a..0e2a9be 100644 --- a/src/datapilot/cli/main.py +++ b/src/datapilot/cli/main.py @@ -50,7 +50,7 @@ def process_config(config): @click.group() -@click.option("--token", required=False, help="Your API token for authentication.") +@click.option("--token", required=False, help="Your API token for authentication.", hide_input=True) @click.option("--instance-name", required=False, help="Your tenant ID.") @click.option("--backend-url", required=False, help="Altimate's Backend URL", default="https://api.myaltimate.com") @click.pass_context From c4059c73d95406da23b5e021584f26b30d871532 Mon Sep 17 00:00:00 2001 From: Michiel De Smet Date: Tue, 10 Jun 2025 13:56:19 +0800 Subject: [PATCH 6/8] chore: update config file loading to use Path.open --- setup.py | 0 src/datapilot/cli/main.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 setup.py diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 diff --git a/src/datapilot/cli/main.py b/src/datapilot/cli/main.py index 0e2a9be..4839fc7 100644 --- a/src/datapilot/cli/main.py +++ b/src/datapilot/cli/main.py @@ -18,7 +18,7 @@ def load_config_from_file(): return {} try: - with open(config_path) as f: + with Path.open(config_path) as f: config = json.load(f) return config except (OSError, json.JSONDecodeError) as e: From e5eaab4ea650ac46410eaae2327dc413e9390024 Mon Sep 17 00:00:00 2001 From: Michiel De Smet Date: Tue, 10 Jun 2025 14:00:34 +0800 Subject: [PATCH 7/8] fix: add 'dbt' prefix to project-health command in tests --- tests/core/platform/dbt/test_cli.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/core/platform/dbt/test_cli.py b/tests/core/platform/dbt/test_cli.py index c79fc72..b7f6f45 100644 --- a/tests/core/platform/dbt/test_cli.py +++ b/tests/core/platform/dbt/test_cli.py @@ -12,7 +12,7 @@ def test_project_health_with_required_and_optional_args(): # Simulate command invocation result = runner.invoke( - datapilot, ["project-health", "--manifest-path", manifest_path, "--catalog-path", catalog_path, "--config-path", config_path] + datapilot, ["dbt", "project-health", "--manifest-path", manifest_path, "--catalog-path", catalog_path, "--config-path", config_path] ) assert result.exit_code == 0 # Ensure the command executed successfully @@ -29,6 +29,7 @@ def test_project_health_with_only_required_arg(): result = runner.invoke( datapilot, [ + "dbt", "project-health", "--manifest-path", manifest_path, @@ -48,6 +49,7 @@ def test_project_health_with_only_required_arg_version1_6(): result = runner.invoke( datapilot, [ + "dbt", "project-health", "--manifest-path", manifest_path, @@ -67,6 +69,7 @@ def test_project_health_with_macro_args(): result = runner.invoke( datapilot, [ + "dbt", "project-health", "--manifest-path", manifest_path, @@ -83,6 +86,7 @@ def test_project_health_with_macro_args(): result = runner.invoke( datapilot, [ + "dbt", "project-health", "--manifest-path", manifest_path, @@ -102,7 +106,7 @@ def test_project_health_with_required_and_optional_args_v12(): # Simulate command invocation result = runner.invoke( - datapilot, ["project-health", "--manifest-path", manifest_path, "--catalog-path", catalog_path, "--config-path", config_path] + datapilot, ["dbt", "project-health", "--manifest-path", manifest_path, "--catalog-path", catalog_path, "--config-path", config_path] ) assert result.exit_code == 0 # Ensure the command executed successfully From a23748faeb8bea6b6d8393b151a563fb8f85928f Mon Sep 17 00:00:00 2001 From: Michiel De Smet Date: Tue, 10 Jun 2025 14:01:51 +0800 Subject: [PATCH 8/8] refactor: simplify config file opening syntax --- src/datapilot/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datapilot/cli/main.py b/src/datapilot/cli/main.py index 4839fc7..a58c2cb 100644 --- a/src/datapilot/cli/main.py +++ b/src/datapilot/cli/main.py @@ -18,7 +18,7 @@ def load_config_from_file(): return {} try: - with Path.open(config_path) as f: + with config_path.open() as f: config = json.load(f) return config except (OSError, json.JSONDecodeError) as e: