Merge pull request #90 from palmetto/release/v0.8.0

Release v0.8.0

.github/CODEOWNERS

@@ -1,4 +1,4 @@
 # see https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners for configs
 
 # review PRs
-*    @jakeberesford-palmetto @murphymoulds-palmetto @mariahjrogers @evandegler-palmetto @omoorepalmetto 
+*    @palmetto/data-analytics @mariahjrogers

CHANGELOG.md

@@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.8.0] - 2023-11-20
+
+### Added
+
+- dbt version detection: palm can now detect the version of dbt used in a project
+  and will add the version to the plugin config. This can then be used to support
+  changes between dbt versions.
+- support for dbt's yaml selectors via the `--selector` option on relevant commands.
+
+### Changed
+
+- Requires palm v2.6.0 or higher
+- env vars are updated for projects running dbt v1.5.0 and later. This resolves
+  deprecation warnings for people using more recent versions of dbt.
+
 ## [0.7.0] - 2023-06-13
 
 ### Added

palm/plugins/dbt/commands/cmd_compile.py

@@ -6,12 +6,14 @@
 @click.command("compile")
 @click.option("--models", "-m", multiple=True, help="See dbt docs on models flag")
 @click.option("--select", "-s", multiple=True, help="See dbt docs on select flag")
+@click.option("--selector", multiple=True, help="See dbt docs on selector flag")
 @click.option("--exclude", "-e", multiple=True, help="See dbt docs on exclude flag")
 @click.pass_obj
 def cli(
     environment,
     models: Optional[Tuple] = tuple(),
     select: Optional[Tuple] = tuple(),
+    selector: Optional[Tuple] = tuple(),
     exclude: Optional[Tuple] = tuple(),
 ):
     """Compiles the dbt repo"""
@@ -24,6 +26,9 @@ def cli(
     if targets:
         cmd.append("--select")
         cmd.extend(targets)
+    if selector:
+        cmd.append("--selector")
+        cmd.extend(selector)
     if exclude:
         cmd.append('--exclude')
         cmd.extend(exclude)

palm/plugins/dbt/commands/cmd_run.py

@@ -14,6 +14,7 @@
 @click.option("--clean", is_flag=True, help="Drop the test schema after the run")
 @click.option("--models", "-m", multiple=True, help="See dbt docs on models flag")
 @click.option("--select", "-s", multiple=True, help="See dbt docs on select flag")
+@click.option("--selector", multiple=True, help="See dbt docs on selector flag")
 @click.option("--exclude", "-e", multiple=True, help="See dbt docs on exclude flag")
 @click.option("--defer", "-d", is_flag=True, help="See dbt docs on defer flag")
 @click.option("--iterative", "-i", is_flag=True, help="Iterative stateful dbt run")
@@ -24,6 +25,7 @@
     help="Will perform a full refresh on incremental models",
 )
 @click.option("--seed", is_flag=True, help="Run dbt seed before dbt run")
+@click.option("--lightdash", '-ld', is_flag=True, help="Run dbt with lightdash enabled")
 @click.pass_obj
 def cli(
     environment,
@@ -33,8 +35,10 @@ def cli(
     iterative: bool,
     defer: bool,
     seed: bool,
+    lightdash: bool,
     models: Optional[Tuple] = tuple(),
     select: Optional[Tuple] = tuple(),
+    selector: Optional[Tuple] = tuple(),
     exclude: Optional[Tuple] = tuple(),
     vars: Optional[str] = None,
 ):
@@ -67,9 +71,11 @@ def cli(
         seed=seed,
         no_fail_fast=(no_fail_fast or iterative),
         targets=targets,
+        selector=selector,
         exclude=exclude,
         defer=defer,
         vars=vars,
+        lightdash=lightdash,
     )
 
     success, msg = environment.run_in_docker(run_cmd, env_vars)
@@ -104,9 +110,11 @@ def build_run_command(
     seed: bool = True,
     no_fail_fast: bool = False,
     targets: Optional[list] = None,
+    selector: Optional[Tuple] = None,
     exclude: Optional[Tuple] = None,
     defer: bool = False,
     vars: Optional[str] = None,
+    lightdash: bool = False,
 ) -> str:
     cmd = []
     full_refresh_option = " --full-refresh" if full_refresh else ""
@@ -115,10 +123,17 @@ def build_run_command(
         cmd.append(f"dbt seed --full-refresh")
         cmd.append("&&")
 
-    cmd.append(f"dbt run{full_refresh_option}")
+    if lightdash:
+        cmd.append(f"lightdash dbt run{full_refresh_option}")
+    else:
+        cmd.append(f"dbt run{full_refresh_option}")
+
     if targets:
         cmd.append("--select")
         cmd.extend(targets)
+    if selector:
+        cmd.append("--selector")
+        cmd.extend(selector)
     if exclude:
         cmd.append("--exclude")
         cmd.extend(exclude)
@@ -137,10 +152,19 @@ def build_run_command(
 def set_env_vars(environment, stateful: bool, defer: bool = False) -> dict:
     plugin_config = environment.plugin_config('dbt')
     env_vars = dbt_env_vars(environment.palm.branch)
+
+    # These env vars are renamed in dbt v1.5.0, old env vars are deprecated
+    if plugin_config.is_dbt_version_greater_than("1.5.0", or_equal=True):
+        defer_env_var = "DBT_DEFER"
+        state_env_var = "DBT_STATE"
+    else:
+        defer_env_var = "DBT_DEFER_TO_STATE"
+        state_env_var = "DBT_ARTIFACT_STATE_PATH"
+
     if stateful:
-        env_vars['DBT_DEFER_TO_STATE'] = 'true'
+        env_vars[defer_env_var] = 'true'
     if defer:
-        env_vars['DBT_ARTIFACT_STATE_PATH'] = plugin_config.dbt_artifacts_prod
+        env_vars[state_env_var] = plugin_config.dbt_artifacts_prod
     else:
-        env_vars['DBT_ARTIFACT_STATE_PATH'] = plugin_config.dbt_artifacts_local
+        env_vars[state_env_var] = plugin_config.dbt_artifacts_local
     return env_vars

palm/plugins/dbt/commands/cmd_seed.py

@@ -8,6 +8,7 @@
     "--clean", is_flag=True, help="drop the test schema after the run is complete"
 )
 @click.option("--select", '-s', multiple=True, help="see dbt docs on select flag")
+@click.option("--selector", multiple=True, help="see dbt docs on selector flag")
 @click.option("--exclude", '-e', multiple=True, help="see dbt docs on exclude flag")
 @click.option(
     "--no-full-refresh",
@@ -21,6 +22,7 @@ def cli(
     clean: bool,
     no_full_refresh: bool,
     select: Optional[Tuple] = tuple(),
+    selector: Optional[Tuple] = tuple(),
     exclude: Optional[Tuple] = tuple(),
 ):
     """Run dbt seeds"""
@@ -29,6 +31,9 @@ def cli(
     if select:
         cmd.append('--select')
         cmd.extend(select)
+    if selector:
+        cmd.append('--selector')
+        cmd.extend(selector)
     if exclude:
         cmd.append('--exclude')
         cmd.extend(exclude)

palm/plugins/dbt/commands/cmd_test.py

@@ -10,6 +10,7 @@
 )
 @click.option("--models", "-m", multiple=True, help="See dbt docs on models flag")
 @click.option("--select", "-s", multiple=True, help="See dbt docs on select flag")
+@click.option("--selector", multiple=True, help="See dbt docs on selector flag")
 @click.option("--exclude", "-e", multiple=True, help="See dbt docs on exclude flag")
 @click.option("--defer", "-d", is_flag=True, help="See dbt docs on defer flag")
 @click.option(
@@ -23,6 +24,7 @@ def cli(
     defer: bool,
     models: Optional[Tuple] = tuple(),
     select: Optional[Tuple] = tuple(),
+    selector: Optional[Tuple] = tuple(),
     exclude: Optional[Tuple] = tuple(),
 ):
     """Tests the dbt repo"""
@@ -50,6 +52,7 @@ def cli(
     run_cmd = build_test_command(
         no_fail_fast=no_fail_fast,
         targets=targets,
+        selector=selector,
         exclude=exclude,
         defer=defer,
     )
@@ -67,6 +70,7 @@ def build_test_command(
     defer: bool = False,
     no_fail_fast: bool = False,
     targets: Optional[list] = None,
+    selector: Optional[Tuple] = None,
     exclude: Optional[Tuple] = None,
 ) -> str:
     cmd = []
@@ -75,6 +79,9 @@ def build_test_command(
     if targets:
         cmd.append('--select')
         cmd.extend(targets)
+    if selector:
+        cmd.append('--selector')
+        cmd.extend(selector)
     if exclude:
         cmd.append('--exclude')
         cmd.extend(exclude)
@@ -91,9 +98,18 @@ def build_test_command(
 def set_env_vars(environment, defer: bool = False) -> dict:
     plugin_config = environment.plugin_config('dbt')
     env_vars = dbt_env_vars(environment.palm.branch)
+
+    # These env vars are renamed in dbt v1.5.0, old env vars are deprecated
+    if plugin_config.is_dbt_version_greater_than("1.5.0", or_equal=True):
+        defer_env_var = "DBT_DEFER"
+        state_env_var = "DBT_STATE"
+    else:
+        defer_env_var = "DBT_DEFER_TO_STATE"
+        state_env_var = "DBT_ARTIFACT_STATE_PATH"
+
     if defer:
-        env_vars['DBT_DEFER_TO_STATE'] = 'true'
-        env_vars['DBT_ARTIFACT_STATE_PATH'] = plugin_config.dbt_artifacts_prod
+        env_vars[defer_env_var] = 'true'
+        env_vars[state_env_var] = plugin_config.dbt_artifacts_prod
     else:
-        env_vars['DBT_ARTIFACT_STATE_PATH'] = plugin_config.dbt_artifacts_local
+        env_vars[state_env_var] = plugin_config.dbt_artifacts_local
     return env_vars

palm/plugins/dbt/dbt_version_detection.py

@@ -0,0 +1,69 @@
+import click
+from pathlib import Path
+import yaml
+
+from palm.utils import run_in_docker
+
+
+def get_dbt_version() -> str:
+    """Get the dbt version from the dbt-core package in the docker image.
+
+    Returns:
+        str: The dbt version
+    """
+    image_name = _get_image_name()
+    cmd = "pip list | grep dbt-core"
+    success, msg = run_in_docker(cmd, image_name, capture_output=True, silent=True)
+    if not success:
+        raise ValueError(f"Error getting dbt version: {msg}")
+    return msg.strip().split(" ")[-1]
+
+
+def dbt_version_factory() -> str:
+    """Get the dbt version from the dbt-core package in the docker image.
+    Then update the dbt version in the .palm/config.yaml file.
+
+    Returns:
+        str: The dbt version
+    """
+    click.secho("Detecting dbt version...", fg="yellow")
+    version = get_dbt_version()
+    set_dbt_version_in_config(version)
+    click.secho(
+        f"Detected dbt version: {version}. .palm/config.yml updated", fg="green"
+    )
+    return version
+
+
+def set_dbt_version_in_config(version: str) -> None:
+    """Update the dbt version in the .palm/config.yaml file."""
+    palm_config = _get_palm_config()
+    palm_config['plugin_config']['dbt']['dbt_version'] = version
+    _update_palm_config(palm_config)
+
+
+def _get_image_name():
+    """Get the image name from the .palm/config.yaml file."""
+    palm_config = _get_palm_config()
+    return palm_config["image_name"]
+
+
+# Ideally, the 2 functions below would come from palm_config.py in palm-cli
+# However, that class is more complicated than it should be and needs a refactor.
+# So, for now, we have to do this here.
+
+
+def _get_palm_config() -> dict:
+    """Get the palm config from the .palm/config.yaml file."""
+    palm_config_path = Path(".palm/config.yaml")
+    if not palm_config_path.exists():
+        raise ValueError("No .palm/config.yaml file found")
+    return yaml.safe_load(palm_config_path.read_text())
+
+
+def _update_palm_config(config: dict) -> None:
+    """Update the palm config in the .palm/config.yaml file."""
+    palm_config_path = Path(".palm/config.yaml")
+    if not palm_config_path.exists():
+        raise ValueError("No .palm/config.yaml file found")
+    palm_config_path.write_text(yaml.dump(config))

palm/plugins/dbt/plugin_config.py

@@ -2,13 +2,32 @@
 
 from typing import Optional
 from pathlib import Path
-from pydantic import BaseModel
+import yaml
+import semver
+from pydantic import BaseModel, Field
 from palm.plugins.base_plugin_config import BasePluginConfig
+from palm.plugins.dbt.dbt_version_detection import dbt_version_factory, get_dbt_version
 
 
 class dbtPluginConfigModel(BaseModel):
     dbt_artifacts_prod: Optional[str]
     dbt_artifacts_local: str
+    dbt_version: str = Field(default_factory=dbt_version_factory)
+
+    def dbt_version_semver(self) -> str:
+        return semver.Version.parse(self.dbt_version)
+
+    def is_dbt_version_greater_than(self, version: str, or_equal: bool = True) -> bool:
+        target_version = semver.Version.parse(version)
+        if or_equal:
+            return self.dbt_version_semver() >= target_version
+        return self.dbt_version_semver() > target_version
+
+    def is_dbt_version_less_than(self, version: str, or_equal: bool = True) -> bool:
+        target_version = semver.Version.parse(version)
+        if or_equal:
+            return self.dbt_version_semver() <= target_version
+        return self.dbt_version_semver() < target_version
 
 
 class DbtPluginConfig(BasePluginConfig):
@@ -23,7 +42,9 @@ def set(self) -> dict:
         )
         if has_prod_artifacts:
             prod = click.prompt(
-                "Prod artifacts location:", type=click.Path(exists=True)
+                "Prod artifacts location:",
+                type=click.Path(exists=True),
+                default=Path("target/prod_artifacts"),
             )
             config["dbt_artifacts_prod"] = str(prod)
             click.secho(f"Saved prod artifacts location:  {prod}", fg="green")
@@ -35,4 +56,6 @@ def set(self) -> dict:
         )
         config['dbt_artifacts_local'] = str(local)
 
+        config['dbt_version'] = get_dbt_version()
+
         return config

palm/plugins/dbt/requirements.txt

@@ -1,3 +1,4 @@
-palm >=2.5.1, <3.0
+palm >=2.6.0, <3.0
 pyyaml >= 5.0
 sqlparse >= 0.3.1
+semver >= 3.0.1

setup.py

@@ -21,7 +21,7 @@
 
 setup(
     name='palm-dbt',
-    version='0.7.0',
+    version='0.8.0',
     description='dbt extension for Palm CLI',
     long_description=long_description,
     long_description_content_type='text/markdown',