Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add config name support #50

Merged
merged 4 commits into from
Mar 7, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions docs/features.rst
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,25 @@ You can also select specific list of models to run the health check on by provid

datapilot dbt project-health --manifest-path ./target/manifest.json --select "path:dir1 path:dir2 model1 model2"

This will run the health check on all the models in the 'dir1' and 'dir2' directory. It will also run the health check on the 'model1' and 'model2' models.
This will run the health check on all the models in the 'dir1' and 'dir2' directory, as well as the 'model1' and 'model2' models.
As of now, the '--select' flag only supports filtering based on model path and model name. We will add support for other filters and make it compatible
with the dbt comands soon.
with the dbt commands soon.

3. **Configuration**:
You can provide configuration in two ways:

a. Using a local config file:
.. code-block:: shell

datapilot dbt project-health --manifest-path ./target/manifest.json --config-path ./path/to/config.yml

b. Using a named config from the API:
.. code-block:: shell

datapilot dbt project-health --manifest-path ./target/manifest.json --config-name "my-config" --token "YOUR_API_TOKEN" --instance-name "YOUR_INSTANCE"

The ``--config-name`` option allows you to use a configuration stored in the Altimate API. When using this option, you must also provide:
- ``--token``: Your API token for authentication
- ``--instance-name``: Your tenant ID

If both ``--config-path`` and ``--config-name`` are provided, the local config file (``--config-path``) takes precedence.
6 changes: 6 additions & 0 deletions src/datapilot/clients/altimate/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,3 +104,9 @@ def run_project_governance_llm_checks(self, manifest, catalog, check_names):
"check_names": check_names,
}
return self.post(endpoint, data=data)

def get_all_dbt_configs(self):
"""Get all DBT configs with a page size of 100."""
endpoint = "/dbtconfig/"
params = {"size": 100}
return self.get(endpoint, params=params)
10 changes: 10 additions & 0 deletions src/datapilot/clients/altimate/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,3 +124,13 @@ def run_project_governance_llm_checks(
):
api_client = APIClient(api_token=api_token, base_url=backend_url, tenant=tenant)
return api_client.run_project_governance_llm_checks(manifest, catalog, check_names)


def get_all_dbt_configs(
api_token,
tenant,
backend_url,
):
"""Get all DBT configs from the API."""
api_client = APIClient(api_token=api_token, base_url=backend_url, tenant=tenant)
return api_client.get_all_dbt_configs()
32 changes: 31 additions & 1 deletion src/datapilot/core/platforms/dbt/cli/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import click

from datapilot.clients.altimate.utils import check_token_and_instance
from datapilot.clients.altimate.utils import get_all_dbt_configs
from datapilot.clients.altimate.utils import onboard_file
from datapilot.clients.altimate.utils import start_dbt_ingestion
from datapilot.clients.altimate.utils import validate_credentials
Expand Down Expand Up @@ -45,6 +46,11 @@ def dbt():
required=False,
help="Path to the DBT config file",
)
@click.option(
"--config-name",
required=False,
help="Name of the DBT config to use from the API",
)
@click.option(
"--select",
required=False,
Expand All @@ -53,7 +59,14 @@ def dbt():
)
@click.option("--backend-url", required=False, help="Altimate's Backend URL", default="https://api.myaltimate.com")
def project_health(
token, instance_name, manifest_path, catalog_path, config_path=None, select=None, backend_url="https://api.myaltimate.com"
token,
instance_name,
manifest_path,
catalog_path,
config_path=None,
config_name=None,
select=None,
backend_url="https://api.myaltimate.com",
):
"""
Validate the DBT project's configuration and structure.
Expand All @@ -62,6 +75,23 @@ def project_health(
config = None
if config_path:
config = load_config(config_path)
elif config_name and token and instance_name:
# Get configs from API
configs = get_all_dbt_configs(token, instance_name, backend_url)
if configs and "items" in configs:
# Find config by name
matching_configs = [c for c in configs["items"] if c["name"] == config_name]
if matching_configs:
# Get the config directly from the API response
click.echo(f"Using config: {config_name}")
config = matching_configs[0].get("config", {})
else:
click.echo(f"No config found with name: {config_name}")
return
else:
click.echo("Failed to fetch configs from API")
return

selected_models = []
if select:
selected_models = select.split(" ")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ def _get_node(self, node: ManifestNode) -> AltimateManifestNode:
contract=contract,
meta=node.meta,
patch_path=node.patch_path,
access=node.access.value,
access=getattr(node.access, "value", None) if hasattr(node, "access") and node.access is not None else None,
)

def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ def _get_node(self, node: ManifestNode) -> AltimateManifestNode:
contract=contract,
meta=node.meta,
patch_path=node.patch_path,
access=node.access.value,
access=getattr(node.access, "value", None) if hasattr(node, "access") and node.access is not None else None,
)

def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ def _get_node(self, node: ManifestNode) -> AltimateManifestNode:
contract=contract,
meta=node.meta,
patch_path=node.patch_path,
access=node.access.value,
access=getattr(node.access, "value", None) if hasattr(node, "access") and node.access is not None else None,
)

def _get_source(self, source: SourceNode) -> AltimateManifestSourceNode:
Expand Down