Merge pull request #89 from jakeberesford-palmetto/feature/fix-deprec…

…ation-warnings

Feature: dbt Version detection & fix deprecation warnings in >= v1.5

palm/plugins/dbt/commands/cmd_run.py

@@ -25,6 +25,7 @@
     help="Will perform a full refresh on incremental models",
 )
 @click.option("--seed", is_flag=True, help="Run dbt seed before dbt run")
+@click.option("--lightdash", '-ld', is_flag=True, help="Run dbt with lightdash enabled")
 @click.pass_obj
 def cli(
     environment,
@@ -34,6 +35,7 @@ def cli(
     iterative: bool,
     defer: bool,
     seed: bool,
+    lightdash: bool,
     models: Optional[Tuple] = tuple(),
     select: Optional[Tuple] = tuple(),
     selector: Optional[Tuple] = tuple(),
@@ -73,6 +75,7 @@ def cli(
         exclude=exclude,
         defer=defer,
         vars=vars,
+        lightdash=lightdash,
     )
 
     success, msg = environment.run_in_docker(run_cmd, env_vars)
@@ -111,6 +114,7 @@ def build_run_command(
     exclude: Optional[Tuple] = None,
     defer: bool = False,
     vars: Optional[str] = None,
+    lightdash: bool = False,
 ) -> str:
     cmd = []
     full_refresh_option = " --full-refresh" if full_refresh else ""
@@ -119,7 +123,11 @@ def build_run_command(
         cmd.append(f"dbt seed --full-refresh")
         cmd.append("&&")
 
-    cmd.append(f"dbt run{full_refresh_option}")
+    if lightdash:
+        cmd.append(f"lightdash dbt run{full_refresh_option}")
+    else:
+        cmd.append(f"dbt run{full_refresh_option}")
+
     if targets:
         cmd.append("--select")
         cmd.extend(targets)
@@ -144,10 +152,19 @@ def build_run_command(
 def set_env_vars(environment, stateful: bool, defer: bool = False) -> dict:
     plugin_config = environment.plugin_config('dbt')
     env_vars = dbt_env_vars(environment.palm.branch)
+
+    # These env vars are renamed in dbt v1.5.0, old env vars are deprecated
+    if plugin_config.is_dbt_version_greater_than("1.5.0", or_equal=True):
+        defer_env_var = "DBT_DEFER"
+        state_env_var = "DBT_STATE"
+    else:
+        defer_env_var = "DBT_DEFER_TO_STATE"
+        state_env_var = "DBT_ARTIFACT_STATE_PATH"
+
     if stateful:
-        env_vars['DBT_DEFER_TO_STATE'] = 'true'
+        env_vars[defer_env_var] = 'true'
     if defer:
-        env_vars['DBT_ARTIFACT_STATE_PATH'] = plugin_config.dbt_artifacts_prod
+        env_vars[state_env_var] = plugin_config.dbt_artifacts_prod
     else:
-        env_vars['DBT_ARTIFACT_STATE_PATH'] = plugin_config.dbt_artifacts_local
+        env_vars[state_env_var] = plugin_config.dbt_artifacts_local
     return env_vars

palm/plugins/dbt/commands/cmd_test.py

@@ -98,9 +98,18 @@ def build_test_command(
 def set_env_vars(environment, defer: bool = False) -> dict:
     plugin_config = environment.plugin_config('dbt')
     env_vars = dbt_env_vars(environment.palm.branch)
+
+    # These env vars are renamed in dbt v1.5.0, old env vars are deprecated
+    if plugin_config.is_dbt_version_greater_than("1.5.0", or_equal=True):
+        defer_env_var = "DBT_DEFER"
+        state_env_var = "DBT_STATE"
+    else:
+        defer_env_var = "DBT_DEFER_TO_STATE"
+        state_env_var = "DBT_ARTIFACT_STATE_PATH"
+
     if defer:
-        env_vars['DBT_DEFER_TO_STATE'] = 'true'
-        env_vars['DBT_ARTIFACT_STATE_PATH'] = plugin_config.dbt_artifacts_prod
+        env_vars[defer_env_var] = 'true'
+        env_vars[state_env_var] = plugin_config.dbt_artifacts_prod
     else:
-        env_vars['DBT_ARTIFACT_STATE_PATH'] = plugin_config.dbt_artifacts_local
+        env_vars[state_env_var] = plugin_config.dbt_artifacts_local
     return env_vars

palm/plugins/dbt/dbt_version_detection.py

@@ -0,0 +1,69 @@
+import click
+from pathlib import Path
+import yaml
+
+from palm.utils import run_in_docker
+
+
+def get_dbt_version() -> str:
+    """Get the dbt version from the dbt-core package in the docker image.
+
+    Returns:
+        str: The dbt version
+    """
+    image_name = _get_image_name()
+    cmd = "pip list | grep dbt-core"
+    success, msg = run_in_docker(cmd, image_name, capture_output=True, silent=True)
+    if not success:
+        raise ValueError(f"Error getting dbt version: {msg}")
+    return msg.strip().split(" ")[-1]
+
+
+def dbt_version_factory() -> str:
+    """Get the dbt version from the dbt-core package in the docker image.
+    Then update the dbt version in the .palm/config.yaml file.
+
+    Returns:
+        str: The dbt version
+    """
+    click.secho("Detecting dbt version...", fg="yellow")
+    version = get_dbt_version()
+    set_dbt_version_in_config(version)
+    click.secho(
+        f"Detected dbt version: {version}. .palm/config.yml updated", fg="green"
+    )
+    return version
+
+
+def set_dbt_version_in_config(version: str) -> None:
+    """Update the dbt version in the .palm/config.yaml file."""
+    palm_config = _get_palm_config()
+    palm_config['plugin_config']['dbt']['dbt_version'] = version
+    _update_palm_config(palm_config)
+
+
+def _get_image_name():
+    """Get the image name from the .palm/config.yaml file."""
+    palm_config = _get_palm_config()
+    return palm_config["image_name"]
+
+
+# Ideally, the 2 functions below would come from palm_config.py in palm-cli
+# However, that class is more complicated than it should be and needs a refactor.
+# So, for now, we have to do this here.
+
+
+def _get_palm_config() -> dict:
+    """Get the palm config from the .palm/config.yaml file."""
+    palm_config_path = Path(".palm/config.yaml")
+    if not palm_config_path.exists():
+        raise ValueError("No .palm/config.yaml file found")
+    return yaml.safe_load(palm_config_path.read_text())
+
+
+def _update_palm_config(config: dict) -> None:
+    """Update the palm config in the .palm/config.yaml file."""
+    palm_config_path = Path(".palm/config.yaml")
+    if not palm_config_path.exists():
+        raise ValueError("No .palm/config.yaml file found")
+    palm_config_path.write_text(yaml.dump(config))

palm/plugins/dbt/plugin_config.py

@@ -2,13 +2,32 @@
 
 from typing import Optional
 from pathlib import Path
-from pydantic import BaseModel
+import yaml
+import semver
+from pydantic import BaseModel, Field
 from palm.plugins.base_plugin_config import BasePluginConfig
+from palm.plugins.dbt.dbt_version_detection import dbt_version_factory, get_dbt_version
 
 
 class dbtPluginConfigModel(BaseModel):
     dbt_artifacts_prod: Optional[str]
     dbt_artifacts_local: str
+    dbt_version: str = Field(default_factory=dbt_version_factory)
+
+    def dbt_version_semver(self) -> str:
+        return semver.Version.parse(self.dbt_version)
+
+    def is_dbt_version_greater_than(self, version: str, or_equal: bool = True) -> bool:
+        target_version = semver.Version.parse(version)
+        if or_equal:
+            return self.dbt_version_semver() >= target_version
+        return self.dbt_version_semver() > target_version
+
+    def is_dbt_version_less_than(self, version: str, or_equal: bool = True) -> bool:
+        target_version = semver.Version.parse(version)
+        if or_equal:
+            return self.dbt_version_semver() <= target_version
+        return self.dbt_version_semver() < target_version
 
 
 class DbtPluginConfig(BasePluginConfig):
@@ -23,7 +42,9 @@ def set(self) -> dict:
         )
         if has_prod_artifacts:
             prod = click.prompt(
-                "Prod artifacts location:", type=click.Path(exists=True)
+                "Prod artifacts location:",
+                type=click.Path(exists=True),
+                default=Path("target/prod_artifacts"),
             )
             config["dbt_artifacts_prod"] = str(prod)
             click.secho(f"Saved prod artifacts location:  {prod}", fg="green")
@@ -35,4 +56,6 @@ def set(self) -> dict:
         )
         config['dbt_artifacts_local'] = str(local)
 
+        config['dbt_version'] = get_dbt_version()
+
         return config

palm/plugins/dbt/requirements.txt

@@ -1,3 +1,4 @@
-palm >=2.5.1, <3.0
+palm >=2.6.0, <3.0
 pyyaml >= 5.0
 sqlparse >= 0.3.1
+semver >= 3.0.1