From d282c5390c58fde8150c03e64c3e8c0baf6d70ac Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Fri, 7 Mar 2025 14:52:37 +0530 Subject: [PATCH 1/4] add config name support --- src/datapilot/clients/altimate/client.py | 6 ++++ src/datapilot/clients/altimate/utils.py | 10 +++++++ src/datapilot/core/platforms/dbt/cli/cli.py | 31 ++++++++++++++++++++- 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/datapilot/clients/altimate/client.py b/src/datapilot/clients/altimate/client.py index 8abd088..1fcc78b 100644 --- a/src/datapilot/clients/altimate/client.py +++ b/src/datapilot/clients/altimate/client.py @@ -104,3 +104,9 @@ def run_project_governance_llm_checks(self, manifest, catalog, check_names): "check_names": check_names, } return self.post(endpoint, data=data) + + def get_all_dbt_configs(self): + """Get all DBT configs with a page size of 100.""" + endpoint = "/dbt/v1/configs" + params = {"size": 100} + return self.get(endpoint, params=params) diff --git a/src/datapilot/clients/altimate/utils.py b/src/datapilot/clients/altimate/utils.py index cc4b4fa..dff87be 100644 --- a/src/datapilot/clients/altimate/utils.py +++ b/src/datapilot/clients/altimate/utils.py @@ -124,3 +124,13 @@ def run_project_governance_llm_checks( ): api_client = APIClient(api_token=api_token, base_url=backend_url, tenant=tenant) return api_client.run_project_governance_llm_checks(manifest, catalog, check_names) + + +def get_all_dbt_configs( + api_token, + tenant, + backend_url, +): + """Get all DBT configs from the API.""" + api_client = APIClient(api_token=api_token, base_url=backend_url, tenant=tenant) + return api_client.get_all_dbt_configs() diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index 24558dc..da76f1c 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -3,6 +3,7 @@ import click from datapilot.clients.altimate.utils import check_token_and_instance +from datapilot.clients.altimate.utils import get_all_dbt_configs from datapilot.clients.altimate.utils import onboard_file from datapilot.clients.altimate.utils import start_dbt_ingestion from datapilot.clients.altimate.utils import validate_credentials @@ -45,6 +46,11 @@ def dbt(): required=False, help="Path to the DBT config file", ) +@click.option( + "--config-name", + required=False, + help="Name of the DBT config to use from the API", +) @click.option( "--select", required=False, @@ -53,7 +59,14 @@ def dbt(): ) @click.option("--backend-url", required=False, help="Altimate's Backend URL", default="https://api.myaltimate.com") def project_health( - token, instance_name, manifest_path, catalog_path, config_path=None, select=None, backend_url="https://api.myaltimate.com" + token, + instance_name, + manifest_path, + catalog_path, + config_path=None, + config_name=None, + select=None, + backend_url="https://api.myaltimate.com", ): """ Validate the DBT project's configuration and structure. @@ -62,6 +75,22 @@ def project_health( config = None if config_path: config = load_config(config_path) + elif config_name and token and instance_name: + # Get configs from API + configs = get_all_dbt_configs(token, instance_name, backend_url) + if configs and "items" in configs: + # Find config by name + matching_configs = [c for c in configs["items"] if c["name"] == config_name] + if matching_configs: + # Get the config directly from the API response + config = matching_configs[0].get("config", {}) + else: + click.echo(f"No config found with name: {config_name}") + return + else: + click.echo("Failed to fetch configs from API") + return + selected_models = [] if select: selected_models = select.split(" ") From 7eca8f512b23fe23adb30d787a08b3994e16d277 Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Fri, 7 Mar 2025 15:06:38 +0530 Subject: [PATCH 2/4] changes --- src/datapilot/clients/altimate/client.py | 2 +- src/datapilot/core/platforms/dbt/cli/cli.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/datapilot/clients/altimate/client.py b/src/datapilot/clients/altimate/client.py index 1fcc78b..be21954 100644 --- a/src/datapilot/clients/altimate/client.py +++ b/src/datapilot/clients/altimate/client.py @@ -107,6 +107,6 @@ def run_project_governance_llm_checks(self, manifest, catalog, check_names): def get_all_dbt_configs(self): """Get all DBT configs with a page size of 100.""" - endpoint = "/dbt/v1/configs" + endpoint = "/dbtconfig/" params = {"size": 100} return self.get(endpoint, params=params) diff --git a/src/datapilot/core/platforms/dbt/cli/cli.py b/src/datapilot/core/platforms/dbt/cli/cli.py index da76f1c..8472ebf 100644 --- a/src/datapilot/core/platforms/dbt/cli/cli.py +++ b/src/datapilot/core/platforms/dbt/cli/cli.py @@ -83,6 +83,7 @@ def project_health( matching_configs = [c for c in configs["items"] if c["name"] == config_name] if matching_configs: # Get the config directly from the API response + click.echo(f"Using config: {config_name}") config = matching_configs[0].get("config", {}) else: click.echo(f"No config found with name: {config_name}") From 3933ac6142f24b89635780d44659b9b507dd70af Mon Sep 17 00:00:00 2001 From: Pulkit Gaur Date: Fri, 7 Mar 2025 15:16:05 +0530 Subject: [PATCH 3/4] update doc --- docs/features.rst | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/features.rst b/docs/features.rst index 98b79bd..8749451 100644 --- a/docs/features.rst +++ b/docs/features.rst @@ -42,6 +42,25 @@ You can also select specific list of models to run the health check on by provid datapilot dbt project-health --manifest-path ./target/manifest.json --select "path:dir1 path:dir2 model1 model2" -This will run the health check on all the models in the 'dir1' and 'dir2' directory. It will also run the health check on the 'model1' and 'model2' models. +This will run the health check on all the models in the 'dir1' and 'dir2' directory, as well as the 'model1' and 'model2' models. As of now, the '--select' flag only supports filtering based on model path and model name. We will add support for other filters and make it compatible -with the dbt comands soon. +with the dbt commands soon. + +3. **Configuration**: +You can provide configuration in two ways: + +a. Using a local config file: + .. code-block:: shell + + datapilot dbt project-health --manifest-path ./target/manifest.json --config-path ./path/to/config.yml + +b. Using a named config from the API: + .. code-block:: shell + + datapilot dbt project-health --manifest-path ./target/manifest.json --config-name "my-config" --token "YOUR_API_TOKEN" --instance-name "YOUR_INSTANCE" + +The ``--config-name`` option allows you to use a configuration stored in the Altimate API. When using this option, you must also provide: +- ``--token``: Your API token for authentication +- ``--instance-name``: Your tenant ID + +If both ``--config-path`` and ``--config-name`` are provided, the local config file (``--config-path``) takes precedence. From b691261a2bd539f7fe75bc8b5e49ac5d512e5284 Mon Sep 17 00:00:00 2001 From: Michiel De Smet Date: Fri, 7 Mar 2025 21:03:49 +0800 Subject: [PATCH 4/4] Fix access property --- .../core/platforms/dbt/wrappers/manifest/v10/wrapper.py | 2 +- .../core/platforms/dbt/wrappers/manifest/v11/wrapper.py | 2 +- .../core/platforms/dbt/wrappers/manifest/v12/wrapper.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py b/src/datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py index 40a4abf..4f1d8dc 100644 --- a/src/datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py +++ b/src/datapilot/core/platforms/dbt/wrappers/manifest/v10/wrapper.py @@ -116,7 +116,7 @@ def _get_node(self, node: ManifestNode) -> AltimateManifestNode: contract=contract, meta=node.meta, patch_path=node.patch_path, - access=node.access.value, + access=getattr(node.access, "value", None) if hasattr(node, "access") and node.access is not None else None, ) def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode: diff --git a/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py b/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py index 8acfc0f..64ef8e7 100644 --- a/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py +++ b/src/datapilot/core/platforms/dbt/wrappers/manifest/v11/wrapper.py @@ -116,7 +116,7 @@ def _get_node(self, node: ManifestNode) -> AltimateManifestNode: contract=contract, meta=node.meta, patch_path=node.patch_path, - access=node.access.value, + access=getattr(node.access, "value", None) if hasattr(node, "access") and node.access is not None else None, ) def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode: diff --git a/src/datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py b/src/datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py index b84715f..551f338 100644 --- a/src/datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py +++ b/src/datapilot/core/platforms/dbt/wrappers/manifest/v12/wrapper.py @@ -116,7 +116,7 @@ def _get_node(self, node: ManifestNode) -> AltimateManifestNode: contract=contract, meta=node.meta, patch_path=node.patch_path, - access=node.access.value, + access=getattr(node.access, "value", None) if hasattr(node, "access") and node.access is not None else None, ) def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode: