Merge pull request #82 from palmetto/release/v0.7.0

v0.7.0 release

CHANGELOG.md

@@ -5,6 +5,23 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.7.0] - 2023-06-13
+
+### Added
+
+- `palm test` now supports the `--defer` option, making it easy to run the tests
+for models you have changed in your current branch.
+
+### Changed
+
+- All palm dbt commands have been updated to have a consistent interface with
+`palm run`.
+- Short options are added to many commands to improve usability.
+
+### Fixed
+
+- `dbt seed` command uses the proper decorator to pass the env object.
+
 ## [0.6.0] - 2023-06-02
 
 ### Added

palm/plugins/dbt/commands/cmd_compile.py

@@ -4,15 +4,29 @@
 
 
 @click.command("compile")
-@click.option("--models", multiple=True, help="see dbt docs on models flag")
-@click.pass_context
-def cli(ctx, models: Optional[Tuple] = tuple()):
-    """Cleans up target directory and dependencies, then compiles dbt"""
+@click.option("--models", "-m", multiple=True, help="See dbt docs on models flag")
+@click.option("--select", "-s", multiple=True, help="See dbt docs on select flag")
+@click.option("--exclude", "-e", multiple=True, help="See dbt docs on exclude flag")
+@click.pass_obj
+def cli(
+    environment,
+    models: Optional[Tuple] = tuple(),
+    select: Optional[Tuple] = tuple(),
+    exclude: Optional[Tuple] = tuple(),
+):
+    """Compiles the dbt repo"""
+    env_vars = dbt_env_vars(environment.palm.branch)
 
-    cmd = "dbt compile"
-    if models:
-        cmd += f" --models {' '.join(models)}"
+    # --select and --models are interchangeable on dbt >= v1, combine the lists of selections
+    targets = list(set(models + select))
 
-    env_vars = dbt_env_vars(ctx.obj.palm.branch)
-    success, msg = ctx.obj.run_in_docker(cmd, env_vars)
+    cmd = ["dbt compile"]
+    if targets:
+        cmd.append("--select")
+        cmd.extend(targets)
+    if exclude:
+        cmd.append('--exclude')
+        cmd.extend(exclude)
+
+    success, msg = environment.run_in_docker(" ".join(cmd), env_vars)
     click.secho(msg, fg="green" if success else "red")

palm/plugins/dbt/commands/cmd_cycle.py

@@ -23,7 +23,7 @@ def cli(
     models: Optional[Tuple] = tuple(),
     select: Optional[Tuple] = tuple(),
 ):
-    """Consecutive run-test of the DBT repo. `count` is the number of run/test cycles to execute, defaults to 2"""
+    """Consecutive run-test of the dbt repo. `count` is the number of run/test cycles to execute, defaults to 2"""
 
     def add_models(command: str) -> str:
         if models:

palm/plugins/dbt/commands/cmd_model.py

@@ -11,7 +11,7 @@
 
 @click.group()
 def cli():
-    """DBT model tools"""
+    """dbt model tools"""
     pass
 
 

palm/plugins/dbt/commands/cmd_prod-artifacts.py

@@ -12,13 +12,13 @@ def cli(environment):
     This command is a no-op as the implementation will differ depending on your
     production environment. You should override this command in your own
     project by running `palm override --name prod-artifacts` and then
-    implementing the logic to download your DBT artifacts from production.
+    implementing the logic to download your dbt artifacts from production.
     """
 
     plugin_config = environment.plugin_config("dbt")
     artifact_path = plugin_config.dbt_artifacts_prod
 
-    msg = "No-op command! Please override this command in your own project by running: palm override --name prod-artifacts'"
+    msg = "No-op command! Please override this command in your own project by running: 'palm override --name prod-artifacts'"
 
     env_vars = dbt_env_vars(environment.palm.branch)
     # success, msg = environment.run_in_docker(cmd, env_vars)

palm/plugins/dbt/commands/cmd_run.py

@@ -7,17 +7,19 @@
 @click.command("run")
 @click.option(
     "--no-fail-fast",
+    "-nx",
     is_flag=True,
     help="Turns off --fail-fast. See dbt docs on fail-fast flag.",
 )
 @click.option("--clean", is_flag=True, help="Drop the test schema after the run")
 @click.option("--models", "-m", multiple=True, help="See dbt docs on models flag")
 @click.option("--select", "-s", multiple=True, help="See dbt docs on select flag")
 @click.option("--exclude", "-e", multiple=True, help="See dbt docs on exclude flag")
-@click.option("--defer", is_flag=True, help="See dbt docs on defer flag")
-@click.option("--iterative", is_flag=True, help="Iterative stateful dbt run")
+@click.option("--defer", "-d", is_flag=True, help="See dbt docs on defer flag")
+@click.option("--iterative", "-i", is_flag=True, help="Iterative stateful dbt run")
 @click.option(
     "--full-refresh",
+    "-f",
     is_flag=True,
     help="Will perform a full refresh on incremental models",
 )
@@ -36,7 +38,7 @@ def cli(
     exclude: Optional[Tuple] = tuple(),
     vars: Optional[str] = None,
 ):
-    """Runs the DBT repo."""
+    """Runs the dbt repo."""
     stateful = iterative or defer
 
     if defer:
@@ -92,7 +94,7 @@ def cli(
 
     if clean:
         success, msg = environment.run_in_docker(
-            "dbt run-operation drop_branch_schemas && dbt clean", env_vars
+            "dbt run-operation drop_branch_schemas", env_vars
         )
         click.secho(msg, fg="green" if success else "red")
 

palm/plugins/dbt/commands/cmd_seed.py

@@ -15,7 +15,7 @@
     is_flag=True,
     help="Insert seeds instead of recreating the table",
 )
-@click.obj
+@click.pass_obj
 def cli(
     environment,
     clean: bool,

palm/plugins/dbt/commands/cmd_snapshot.py

@@ -5,17 +5,31 @@
 
 @click.command("snapshot")
 @click.option("--clean", is_flag=True, help="Drop the test schema after the run")
-@click.option("--select", multiple=True)
-@click.pass_context
-def cli(ctx, clean: bool, select: Optional[Tuple] = tuple()):
-    """Executes the DBT snapshots."""
+@click.option("--select", "-s", multiple=True, help="See dbt docs on select flag")
+@click.option("--exclude", "-e", multiple=True, help="See dbt docs on exclude flag")
+@click.pass_obj
+def cli(
+    environment,
+    clean: bool,
+    select: Optional[Tuple] = tuple(),
+    exclude: Optional[Tuple] = tuple(),
+):
+    """Executes the dbt snapshots."""
+    env_vars = dbt_env_vars(environment.palm.branch)
 
-    cmd = "dbt snapshot"
+    cmd = ["dbt snapshot"]
     if select:
-        cmd += f" --select " + " ".join(select)
-    if clean:
-        cmd += " && dbt run-operation drop_branch_schemas"
+        cmd.append("--select")
+        cmd.extend(select)
+    if exclude:
+        cmd.append('--exclude')
+        cmd.extend(exclude)
 
-    env_vars = dbt_env_vars(ctx.obj.palm.branch)
-    success, msg = ctx.obj.run_in_docker(cmd, env_vars)
+    success, msg = environment.run_in_docker(" ".join(cmd), env_vars)
     click.secho(msg, fg="green" if success else "red")
+
+    if clean:
+        success, msg = environment.run_in_docker(
+            "dbt run-operation drop_branch_schemas", env_vars
+        )
+        click.secho(msg, fg="green" if success else "red")

palm/plugins/dbt/commands/cmd_test.py

@@ -1,42 +1,99 @@
 import click
 from typing import Optional, Tuple
 from palm.plugins.dbt.dbt_palm_utils import dbt_env_vars
+import sys
 
 
 @click.command('test')
 @click.option(
-    "--clean", is_flag=True, help="drop the test schema after the run is complete"
+    "--clean", is_flag=True, help="Drop the test schema after the run is complete"
 )
-@click.option("--models", multiple=True, help="see dbt docs on models flag")
-@click.option("--select", multiple=True, help="see dbt docs on select flag")
-@click.option("--exclude", multiple=True, help="see dbt docs on exclude flag")
-@click.option("--no-fail-fast", is_flag=True, help="will run all tests if one fails")
-@click.pass_context
+@click.option("--models", "-m", multiple=True, help="See dbt docs on models flag")
+@click.option("--select", "-s", multiple=True, help="See dbt docs on select flag")
+@click.option("--exclude", "-e", multiple=True, help="See dbt docs on exclude flag")
+@click.option("--defer", "-d", is_flag=True, help="See dbt docs on defer flag")
+@click.option(
+    "--no-fail-fast", "-nx", is_flag=True, help="Runs all tests even if one fails"
+)
+@click.pass_obj
 def cli(
-    ctx,
+    environment,
     clean: bool,
     no_fail_fast: bool,
+    defer: bool,
     models: Optional[Tuple] = tuple(),
     select: Optional[Tuple] = tuple(),
     exclude: Optional[Tuple] = tuple(),
 ):
-    """Tests the DBT repo"""
+    """Tests the dbt repo"""
+
+    if defer:
+        click.secho("Running 'palm prod-artifacts'...", fg='yellow')
+        exit_code, _, _ = environment.run_on_host("palm prod-artifacts")
+        if exit_code == 2:
+            click.secho(
+                "'palm prod-artifacts' not implemented. Can't use --defer without it!",
+                fg='red',
+            )
+            sys.exit(1)
+        elif exit_code != 0:
+            click.secho(
+                "Something went wrong while pulling the prod artifacts.", fg='red'
+            )
+            sys.exit(1)
+
+    env_vars = set_env_vars(environment, defer)
 
-    cmd = ['dbt', 'test']
-    if select:
+    # --select and --models are interchangeable on dbt >= v1, combine the lists of selections
+    targets = list(set(models + select))
+
+    run_cmd = build_test_command(
+        no_fail_fast=no_fail_fast,
+        targets=targets,
+        exclude=exclude,
+        defer=defer,
+    )
+    success, msg = environment.run_in_docker(run_cmd, env_vars)
+    click.secho(msg, fg="green" if success else "red")
+
+    if clean:
+        success, msg = environment.run_in_docker(
+            "dbt run-operation drop_branch_schemas", env_vars
+        )
+        click.secho(msg, fg="green" if success else "red")
+
+
+def build_test_command(
+    defer: bool = False,
+    no_fail_fast: bool = False,
+    targets: Optional[list] = None,
+    exclude: Optional[Tuple] = None,
+) -> str:
+    cmd = []
+
+    cmd.append(f"dbt test")
+    if targets:
         cmd.append('--select')
-        cmd.extend(select)
-    if models:
-        cmd.append('--models')
-        cmd.extend(models)
+        cmd.extend(targets)
     if exclude:
         cmd.append('--exclude')
         cmd.extend(exclude)
     if not no_fail_fast:
         cmd.append('--fail-fast')
-    if clean:
-        cmd.append('&& dbt run-operation drop_branch_schemas')
+    if defer:
+        if not targets:
+            cmd.extend(["--select", "state:new", "state:modified+"])
+        cmd.append("--defer")
 
-    env_vars = dbt_env_vars(ctx.obj.palm.branch)
-    success, msg = ctx.obj.run_in_docker(" ".join(cmd), env_vars)
-    click.secho(msg, fg="green" if success else "red")
+    return " ".join(cmd)
+
+
+def set_env_vars(environment, defer: bool = False) -> dict:
+    plugin_config = environment.plugin_config('dbt')
+    env_vars = dbt_env_vars(environment.palm.branch)
+    if defer:
+        env_vars['DBT_DEFER_TO_STATE'] = 'true'
+        env_vars['DBT_ARTIFACT_STATE_PATH'] = plugin_config.dbt_artifacts_prod
+    else:
+        env_vars['DBT_ARTIFACT_STATE_PATH'] = plugin_config.dbt_artifacts_local
+    return env_vars

palm/plugins/dbt/dbt_palm_utils.py

@@ -2,7 +2,7 @@
 from typing import Dict
 from palm.plugins.dbt.local_user_lookup import local_user_lookup
 
-""" Shared DBT utilities to build out common CLI options """
+""" Shared dbt utilities to build out common CLI options """
 
 
 def dbt_env_vars(branch: str) -> Dict:

setup.py

@@ -21,7 +21,7 @@
 
 setup(
     name='palm-dbt',
-    version='0.6.0',
+    version='0.7.0',
     description='dbt extension for Palm CLI',
     long_description=long_description,
     long_description_content_type='text/markdown',