From 06abf5a8fe47f0096780ebe4fbbd7ad53c37b6c8 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Tue, 7 Dec 2021 14:42:51 -0500 Subject: [PATCH 01/19] dbt deps in docker image by default Running dbt deps all the time is slow and error prone (if dbt hub is down, or there is a github outage). Forgetting to run dbt deps can also cause confusing errors. By changing our baseline implementation to assume dbt deps is run in the docker image, we can avoid these problems. The --deps option has been added to every relevant command this allows a user to specify they want to run deps with a given command * Also refactored some nasty conditional branching in cmd_cycle --- palm/plugins/dbt/commands/cmd_cleanup.py | 8 +++-- palm/plugins/dbt/commands/cmd_compile.py | 12 ++++---- palm/plugins/dbt/commands/cmd_cycle.py | 29 +++++++++---------- palm/plugins/dbt/commands/cmd_run.py | 2 ++ palm/plugins/dbt/commands/cmd_snapshot.py | 11 +++---- palm/plugins/dbt/commands/cmd_test.py | 2 ++ palm/plugins/dbt/dbt_palm_utils.py | 7 +++-- .../dbt/templates/containerize/Dockerfile.txt | 2 ++ 8 files changed, 42 insertions(+), 31 deletions(-) diff --git a/palm/plugins/dbt/commands/cmd_cleanup.py b/palm/plugins/dbt/commands/cmd_cleanup.py index 059f1a6..0ad125b 100644 --- a/palm/plugins/dbt/commands/cmd_cleanup.py +++ b/palm/plugins/dbt/commands/cmd_cleanup.py @@ -5,11 +5,15 @@ @click.command("cleanup") +@click.option("--deps", is_flag=True, help="Will clean and install dependencies") @click.pass_context -def cli(ctx): +def cli(ctx, deps: bool): """Removes any artifacts from Snowflake related to the current branch.""" - cmd = "dbt clean && dbt deps && dbt run-operation drop_branch_schemas" + if deps: + cmd = "dbt clean && dbt deps && dbt run-operation drop_branch_schemas" + else: + cmd = "dbt run-operation drop_branch_schemas" env_vars = dbt_env_vars(ctx.obj.palm.branch) success, msg = ctx.obj.run_in_docker(cmd, env_vars) click.secho(msg, fg="green" if success else "red") diff --git a/palm/plugins/dbt/commands/cmd_compile.py b/palm/plugins/dbt/commands/cmd_compile.py index b103dbf..e7aa3e7 100644 --- a/palm/plugins/dbt/commands/cmd_compile.py +++ b/palm/plugins/dbt/commands/cmd_compile.py @@ -5,17 +5,17 @@ @click.command("compile") @click.option("--models", multiple=True, help="see dbt docs on models flag") -@click.option("--fast", is_flag=True, help="will skip clean/deps/seed") +@click.option("--deps", is_flag=True, help="Will clean and install dependencies") @click.pass_context -def cli(ctx, fast: bool, models: Optional[Tuple] = tuple()): +def cli(ctx, deps: bool, models: Optional[Tuple] = tuple()): """Cleans up target directory and dependencies, then compiles dbt""" - if fast: - cmd = "dbt compile" - else: + if deps: cmd = "dbt clean && dbt deps && dbt compile" + else: + cmd = "dbt compile" if models: - cmd += f" --models " + " ".join(models) + cmd += f" --models {' '.join(models)}" env_vars = dbt_env_vars(ctx.obj.palm.branch) success, msg = ctx.obj.run_in_docker(cmd, env_vars) diff --git a/palm/plugins/dbt/commands/cmd_cycle.py b/palm/plugins/dbt/commands/cmd_cycle.py index 1cbb7d1..0775ba5 100644 --- a/palm/plugins/dbt/commands/cmd_cycle.py +++ b/palm/plugins/dbt/commands/cmd_cycle.py @@ -15,6 +15,7 @@ @click.option("--models", multiple=True, help="see dbt docs on models flag") @click.option("--select", multiple=True, help="see dbt docs on select flag") @click.option("--no-seed", is_flag=True, help="will skip seed full refresh") +@click.option("--deps", is_flag=True, help="clean and install dependencies") @click.pass_context def cli( ctx, @@ -22,6 +23,7 @@ def cli( fast: bool, persist: bool, no_seed: bool, + deps: bool, models: Optional[Tuple] = tuple(), select: Optional[Tuple] = tuple(), ): @@ -53,22 +55,17 @@ def run_test(): ) def make_cmd(): - if fast: - if no_seed: - seed_cmd = "" - else: - seed_cmd = "dbt seed --full-refresh" - commands = [add_select(seed_cmd), run_test(), add_persist()] - else: - if no_seed: - seed_cmd = "" - else: - seed_cmd = " && dbt seed --full-refresh " - commands = [ - add_select("dbt clean && dbt deps" + seed_cmd), - run_test(), - add_persist(), - ] + commands = [] + if not fast or deps: + commands.append("dbt clean && dbt deps") + + if not no_seed: + seed_cmd = "dbt seed --full-refresh" + commands.append(add_select(seed_cmd)) + + commands.append(run_test()) + commands.append(add_persist()()) + return " && ".join(list(filter(None, commands))) env_vars = dbt_env_vars(ctx.obj.palm.branch) diff --git a/palm/plugins/dbt/commands/cmd_run.py b/palm/plugins/dbt/commands/cmd_run.py index 740146f..5b59c3c 100644 --- a/palm/plugins/dbt/commands/cmd_run.py +++ b/palm/plugins/dbt/commands/cmd_run.py @@ -24,6 +24,7 @@ help="will perform a full refresh on incremental models", ) @click.option("--no-seed", is_flag=True, help="will skip seed full refresh") +@click.option("--deps", is_flag=True, help="will run clean and deps") @click.pass_context def cli( ctx, @@ -32,6 +33,7 @@ def cli( persist: bool, full_refresh: bool, no_seed: bool, + deps: bool, models: Optional[Tuple] = tuple(), select: Optional[Tuple] = tuple(), macros: Optional[Tuple] = tuple(), diff --git a/palm/plugins/dbt/commands/cmd_snapshot.py b/palm/plugins/dbt/commands/cmd_snapshot.py index 28cd456..12bf3eb 100644 --- a/palm/plugins/dbt/commands/cmd_snapshot.py +++ b/palm/plugins/dbt/commands/cmd_snapshot.py @@ -8,15 +8,16 @@ "--persist", is_flag=True, help="will not drop the test schema at the end" ) @click.option("--select", multiple=True) -@click.option("--fast", is_flag=True, help="will skip clean/deps/seed") +@click.option("--fast", is_flag=True, help="will skip clean & deps") +@click.option("--deps", is_flag=True, help="will run clean & deps") @click.pass_context -def cli(ctx, persist: bool, fast: bool, select: Optional[Tuple] = tuple()): +def cli(ctx, persist: bool, fast: bool, deps: bool, select: Optional[Tuple] = tuple()): """Executes the DBT snapshots.""" - if fast: - cmd = "dbt snapshot" - else: + if not fast or deps: cmd = "dbt clean && dbt deps && dbt snapshot" + else: + cmd = "dbt snapshot" if select: cmd += f" --select " + " ".join(select) if not persist: diff --git a/palm/plugins/dbt/commands/cmd_test.py b/palm/plugins/dbt/commands/cmd_test.py index 8ed7ca5..e4794eb 100644 --- a/palm/plugins/dbt/commands/cmd_test.py +++ b/palm/plugins/dbt/commands/cmd_test.py @@ -12,6 +12,7 @@ @click.option("--select", multiple=True, help="see dbt docs on select flag") @click.option("--no-seed", is_flag=True, help="will skip seed full refresh") @click.option("--no-fail-fast", is_flag=True, help="will run all tests if one fails") +@click.option("--deps", is_flag=True, help="will run clean and deps") @click.pass_context def cli( ctx, @@ -19,6 +20,7 @@ def cli( persist: bool, no_seed: bool, no_fail_fast: bool, + deps: bool, models: Optional[Tuple] = tuple(), select: Optional[Tuple] = tuple(), ): diff --git a/palm/plugins/dbt/dbt_palm_utils.py b/palm/plugins/dbt/dbt_palm_utils.py index f9a88b4..f4b2293 100644 --- a/palm/plugins/dbt/dbt_palm_utils.py +++ b/palm/plugins/dbt/dbt_palm_utils.py @@ -53,12 +53,15 @@ def _long_cycle( no_fail_fast: Optional[bool] = False, full_refresh: Optional[bool] = False, no_seed: Optional[bool] = False, + deps: Optional[bool] = False, select: Optional[Tuple] = (), models: Optional[Tuple] = (), macros: Optional[Tuple] = (), ) -> str: - seed_cmd = "" if no_seed else "&& dbt seed --full-refresh" - command = f"dbt clean && dbt deps {seed_cmd}" + command = "" if no_seed else "&& dbt seed --full-refresh" + + if deps: + command = f'dbt clean && dbt deps && {command}' if macros: run_operation = " && dbt run-operation " diff --git a/palm/plugins/dbt/templates/containerize/Dockerfile.txt b/palm/plugins/dbt/templates/containerize/Dockerfile.txt index 6c354ed..6cf3d2a 100644 --- a/palm/plugins/dbt/templates/containerize/Dockerfile.txt +++ b/palm/plugins/dbt/templates/containerize/Dockerfile.txt @@ -4,4 +4,6 @@ WORKDIR /app ENV PYTHONPATH=${PYTHONPATH}:${PWD} RUN ./scripts/entrypoint.sh +RUN dbt clean && dbt deps + ENTRYPOINT [ "" ] From be46e85151e42383761012e4a7dc04dfae1434d2 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Tue, 7 Dec 2021 14:50:56 -0500 Subject: [PATCH 02/19] One pair of parens is enough. --- palm/plugins/dbt/commands/cmd_cycle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/palm/plugins/dbt/commands/cmd_cycle.py b/palm/plugins/dbt/commands/cmd_cycle.py index 0775ba5..08f8424 100644 --- a/palm/plugins/dbt/commands/cmd_cycle.py +++ b/palm/plugins/dbt/commands/cmd_cycle.py @@ -64,8 +64,8 @@ def make_cmd(): commands.append(add_select(seed_cmd)) commands.append(run_test()) - commands.append(add_persist()()) - + commands.append(add_persist()) + return " && ".join(list(filter(None, commands))) env_vars = dbt_env_vars(ctx.obj.palm.branch) From 7b5862743bf8cbed49f389a3f9e47d4813a27979 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Tue, 7 Dec 2021 15:38:23 -0500 Subject: [PATCH 03/19] Remove the --fast flag All commands are fast once dbt clean && deps happens in the image! --- README.md | 2 +- palm/plugins/dbt/commands/cmd_cleanup.py | 8 +-- palm/plugins/dbt/commands/cmd_compile.py | 8 +-- palm/plugins/dbt/commands/cmd_cycle.py | 7 --- palm/plugins/dbt/commands/cmd_run.py | 4 -- palm/plugins/dbt/commands/cmd_snapshot.py | 9 +--- palm/plugins/dbt/commands/cmd_test.py | 4 -- palm/plugins/dbt/dbt_palm_utils.py | 66 +++++------------------ 8 files changed, 21 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index bf49688..131fbbe 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ for the same developer (like during hotfixes). From a non-protected branch, running `palm run` will: 1. drop (if it exists) the namespaced schema in development 2. create the namespaced schema in development -3. clean, deps, seed and run +3. seed and run 4. drop the namespaced schema in development Why drop it? so your testing is atomic. diff --git a/palm/plugins/dbt/commands/cmd_cleanup.py b/palm/plugins/dbt/commands/cmd_cleanup.py index 0ad125b..88aae76 100644 --- a/palm/plugins/dbt/commands/cmd_cleanup.py +++ b/palm/plugins/dbt/commands/cmd_cleanup.py @@ -5,15 +5,11 @@ @click.command("cleanup") -@click.option("--deps", is_flag=True, help="Will clean and install dependencies") @click.pass_context -def cli(ctx, deps: bool): +def cli(ctx): """Removes any artifacts from Snowflake related to the current branch.""" - if deps: - cmd = "dbt clean && dbt deps && dbt run-operation drop_branch_schemas" - else: - cmd = "dbt run-operation drop_branch_schemas" + cmd = "dbt run-operation drop_branch_schemas" env_vars = dbt_env_vars(ctx.obj.palm.branch) success, msg = ctx.obj.run_in_docker(cmd, env_vars) click.secho(msg, fg="green" if success else "red") diff --git a/palm/plugins/dbt/commands/cmd_compile.py b/palm/plugins/dbt/commands/cmd_compile.py index e7aa3e7..538fc56 100644 --- a/palm/plugins/dbt/commands/cmd_compile.py +++ b/palm/plugins/dbt/commands/cmd_compile.py @@ -5,15 +5,11 @@ @click.command("compile") @click.option("--models", multiple=True, help="see dbt docs on models flag") -@click.option("--deps", is_flag=True, help="Will clean and install dependencies") @click.pass_context -def cli(ctx, deps: bool, models: Optional[Tuple] = tuple()): +def cli(ctx, models: Optional[Tuple] = tuple()): """Cleans up target directory and dependencies, then compiles dbt""" - if deps: - cmd = "dbt clean && dbt deps && dbt compile" - else: - cmd = "dbt compile" + cmd = "dbt compile" if models: cmd += f" --models {' '.join(models)}" diff --git a/palm/plugins/dbt/commands/cmd_cycle.py b/palm/plugins/dbt/commands/cmd_cycle.py index 08f8424..815d335 100644 --- a/palm/plugins/dbt/commands/cmd_cycle.py +++ b/palm/plugins/dbt/commands/cmd_cycle.py @@ -8,22 +8,18 @@ @click.command("cycle") @click.argument("count", type=int, default=2) -@click.option("--fast", is_flag=True, help="will skip clean/deps/seed") @click.option( "--persist", is_flag=True, help="will not drop the test schema at the end" ) @click.option("--models", multiple=True, help="see dbt docs on models flag") @click.option("--select", multiple=True, help="see dbt docs on select flag") @click.option("--no-seed", is_flag=True, help="will skip seed full refresh") -@click.option("--deps", is_flag=True, help="clean and install dependencies") @click.pass_context def cli( ctx, count: int, - fast: bool, persist: bool, no_seed: bool, - deps: bool, models: Optional[Tuple] = tuple(), select: Optional[Tuple] = tuple(), ): @@ -56,9 +52,6 @@ def run_test(): def make_cmd(): commands = [] - if not fast or deps: - commands.append("dbt clean && dbt deps") - if not no_seed: seed_cmd = "dbt seed --full-refresh" commands.append(add_select(seed_cmd)) diff --git a/palm/plugins/dbt/commands/cmd_run.py b/palm/plugins/dbt/commands/cmd_run.py index 5b59c3c..b22c8fb 100644 --- a/palm/plugins/dbt/commands/cmd_run.py +++ b/palm/plugins/dbt/commands/cmd_run.py @@ -4,7 +4,6 @@ @click.command("run") -@click.option("--fast", is_flag=True, help="will skip clean/deps/seed") @click.option( "--no-fail-fast", is_flag=True, @@ -24,16 +23,13 @@ help="will perform a full refresh on incremental models", ) @click.option("--no-seed", is_flag=True, help="will skip seed full refresh") -@click.option("--deps", is_flag=True, help="will run clean and deps") @click.pass_context def cli( ctx, - fast: bool, no_fail_fast: bool, persist: bool, full_refresh: bool, no_seed: bool, - deps: bool, models: Optional[Tuple] = tuple(), select: Optional[Tuple] = tuple(), macros: Optional[Tuple] = tuple(), diff --git a/palm/plugins/dbt/commands/cmd_snapshot.py b/palm/plugins/dbt/commands/cmd_snapshot.py index 12bf3eb..9147ae3 100644 --- a/palm/plugins/dbt/commands/cmd_snapshot.py +++ b/palm/plugins/dbt/commands/cmd_snapshot.py @@ -8,16 +8,11 @@ "--persist", is_flag=True, help="will not drop the test schema at the end" ) @click.option("--select", multiple=True) -@click.option("--fast", is_flag=True, help="will skip clean & deps") -@click.option("--deps", is_flag=True, help="will run clean & deps") @click.pass_context -def cli(ctx, persist: bool, fast: bool, deps: bool, select: Optional[Tuple] = tuple()): +def cli(ctx, persist: bool, select: Optional[Tuple] = tuple()): """Executes the DBT snapshots.""" - if not fast or deps: - cmd = "dbt clean && dbt deps && dbt snapshot" - else: - cmd = "dbt snapshot" + cmd = "dbt snapshot" if select: cmd += f" --select " + " ".join(select) if not persist: diff --git a/palm/plugins/dbt/commands/cmd_test.py b/palm/plugins/dbt/commands/cmd_test.py index e4794eb..288f962 100644 --- a/palm/plugins/dbt/commands/cmd_test.py +++ b/palm/plugins/dbt/commands/cmd_test.py @@ -4,7 +4,6 @@ @click.command('test') -@click.option("--fast", is_flag=True, help="will skip clean/deps/seed") @click.option( "--persist", is_flag=True, help="will not drop the test schema at the end" ) @@ -12,15 +11,12 @@ @click.option("--select", multiple=True, help="see dbt docs on select flag") @click.option("--no-seed", is_flag=True, help="will skip seed full refresh") @click.option("--no-fail-fast", is_flag=True, help="will run all tests if one fails") -@click.option("--deps", is_flag=True, help="will run clean and deps") @click.pass_context def cli( ctx, - fast: bool, persist: bool, no_seed: bool, no_fail_fast: bool, - deps: bool, models: Optional[Tuple] = tuple(), select: Optional[Tuple] = tuple(), ): diff --git a/palm/plugins/dbt/dbt_palm_utils.py b/palm/plugins/dbt/dbt_palm_utils.py index f4b2293..e39021a 100644 --- a/palm/plugins/dbt/dbt_palm_utils.py +++ b/palm/plugins/dbt/dbt_palm_utils.py @@ -9,78 +9,40 @@ def shell_options(command_name: str, **kwargs): # pop off unused kwargs caused by use of importlib in palm kwargs.pop("ctx") - if kwargs["fast"]: - for k in ("fast", "select"): - kwargs.pop(k, None) - return _short_cycle(command_name, **kwargs) - return - kwargs.pop("fast") - return _long_cycle(command_name, **kwargs) + return _cycle(command_name, **kwargs) - -def _short_cycle( - cmd: str, - persist: Optional[bool] = False, - no_fail_fast: Optional[bool] = False, - full_refresh: Optional[bool] = False, - no_seed: Optional[bool] = False, - models: Optional[Tuple] = (), - macros: Optional[Tuple] = (), -) -> str: - - command = "" if no_seed else f" dbt seed --full-refresh && " - - if macros: - command += f"dbt run-operation {macros}" - else: - command += f"dbt {cmd}" - if models: - command += " --models " + " ".join(models) - if full_refresh: - command += " --full-refresh" - if not no_fail_fast: - command += " --fail-fast" - - if not persist: - command += " && dbt run-operation drop_branch_schemas" - - return command - - -def _long_cycle( +def _cycle( cmd: str, persist: Optional[bool] = False, no_fail_fast: Optional[bool] = False, full_refresh: Optional[bool] = False, no_seed: Optional[bool] = False, - deps: Optional[bool] = False, select: Optional[Tuple] = (), models: Optional[Tuple] = (), macros: Optional[Tuple] = (), ) -> str: - command = "" if no_seed else "&& dbt seed --full-refresh" - - if deps: - command = f'dbt clean && dbt deps && {command}' + command = [] + if not no_seed: + command.append("dbt seed --full-refresh") if macros: - run_operation = " && dbt run-operation " - command += run_operation + run_operation.join(macros) + command.append(f'dbt run-operation + {"&& dbt run-operation".join(macros)}') else: - command += f" && dbt {cmd}" + subcommand = f"dbt {cmd}" if select and not no_seed: - command += " --select " + " ".join(select) + subcommand += " --select " + " ".join(select) if models: - command += " --models " + " ".join(models) + subcommand += " --models " + " ".join(models) if full_refresh: - command += " --full-refresh" + subcommand += " --full-refresh" if not no_fail_fast: - command += " --fail-fast" + subcommand += " --fail-fast" + command.append(subcommand) if not persist: - command += " && dbt run-operation drop_branch_schemas" + command.append("dbt run-operation drop_branch_schemas") - return command + return ' && '.join(command) def dbt_env_vars(branch: str) -> Dict: From 2dfd71ea6f62ee9142be163fd070aaacb968691a Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Tue, 7 Dec 2021 15:42:14 -0500 Subject: [PATCH 04/19] Add new command to recompile dbt deps --- palm/plugins/dbt/commands/cmd_deps.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 palm/plugins/dbt/commands/cmd_deps.py diff --git a/palm/plugins/dbt/commands/cmd_deps.py b/palm/plugins/dbt/commands/cmd_deps.py new file mode 100644 index 0000000..3a49495 --- /dev/null +++ b/palm/plugins/dbt/commands/cmd_deps.py @@ -0,0 +1,10 @@ +import click +from palm.plugins.dbt.dbt_palm_utils import dbt_env_vars + + +@click.command('deps') +@click.pass_context +def cli(ctx): + """Recompile dbt deps""" + env_vars = dbt_env_vars(ctx.obj.palm.branch) + ctx.obj.run_in_docker('dbt deps', env_vars) From a258f5e72d4370b681a38fe1e0c04a9a8dc689cf Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Tue, 7 Dec 2021 15:52:14 -0500 Subject: [PATCH 05/19] Run linting --- palm/plugins/dbt/commands/cmd_cycle.py | 2 +- palm/plugins/dbt/dbt_palm_utils.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/palm/plugins/dbt/commands/cmd_cycle.py b/palm/plugins/dbt/commands/cmd_cycle.py index 815d335..041255b 100644 --- a/palm/plugins/dbt/commands/cmd_cycle.py +++ b/palm/plugins/dbt/commands/cmd_cycle.py @@ -55,7 +55,7 @@ def make_cmd(): if not no_seed: seed_cmd = "dbt seed --full-refresh" commands.append(add_select(seed_cmd)) - + commands.append(run_test()) commands.append(add_persist()) diff --git a/palm/plugins/dbt/dbt_palm_utils.py b/palm/plugins/dbt/dbt_palm_utils.py index e39021a..72cb36a 100644 --- a/palm/plugins/dbt/dbt_palm_utils.py +++ b/palm/plugins/dbt/dbt_palm_utils.py @@ -11,6 +11,7 @@ def shell_options(command_name: str, **kwargs): return _cycle(command_name, **kwargs) + def _cycle( cmd: str, persist: Optional[bool] = False, From 772f08d2d75b62560b18a632890029dbe83b2bd3 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Tue, 7 Dec 2021 16:04:38 -0500 Subject: [PATCH 06/19] We still want to run clean on palm cleanup --- palm/plugins/dbt/commands/cmd_cleanup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/palm/plugins/dbt/commands/cmd_cleanup.py b/palm/plugins/dbt/commands/cmd_cleanup.py index 88aae76..6d612bf 100644 --- a/palm/plugins/dbt/commands/cmd_cleanup.py +++ b/palm/plugins/dbt/commands/cmd_cleanup.py @@ -9,7 +9,7 @@ def cli(ctx): """Removes any artifacts from Snowflake related to the current branch.""" - cmd = "dbt run-operation drop_branch_schemas" + cmd = "dbt clean && dbt run-operation drop_branch_schemas" env_vars = dbt_env_vars(ctx.obj.palm.branch) success, msg = ctx.obj.run_in_docker(cmd, env_vars) click.secho(msg, fg="green" if success else "red") From c5fcfc9938ae27192eeb31ffb5ea36e63c02413f Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Tue, 7 Dec 2021 18:15:31 -0500 Subject: [PATCH 07/19] Drop branch schemas before clean to ensure all branches are removed Run dbt deps after dbt clean as we will always need to re-instlal deps at this point --- palm/plugins/dbt/commands/cmd_cleanup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/palm/plugins/dbt/commands/cmd_cleanup.py b/palm/plugins/dbt/commands/cmd_cleanup.py index 6d612bf..e2f8ce5 100644 --- a/palm/plugins/dbt/commands/cmd_cleanup.py +++ b/palm/plugins/dbt/commands/cmd_cleanup.py @@ -9,7 +9,7 @@ def cli(ctx): """Removes any artifacts from Snowflake related to the current branch.""" - cmd = "dbt clean && dbt run-operation drop_branch_schemas" + cmd = "dbt run-operation drop_branch_schemas && dbt clean && dbt deps" env_vars = dbt_env_vars(ctx.obj.palm.branch) success, msg = ctx.obj.run_in_docker(cmd, env_vars) click.secho(msg, fg="green" if success else "red") From f38350bb7653e62e57ea9d667b23e873d6e5b3c0 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Wed, 8 Dec 2021 11:19:53 -0500 Subject: [PATCH 08/19] add_persist should return an empty string unless not persist this was likely working by accident --- palm/plugins/dbt/commands/cmd_cycle.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/palm/plugins/dbt/commands/cmd_cycle.py b/palm/plugins/dbt/commands/cmd_cycle.py index 041255b..f191d2d 100644 --- a/palm/plugins/dbt/commands/cmd_cycle.py +++ b/palm/plugins/dbt/commands/cmd_cycle.py @@ -37,8 +37,8 @@ def add_select(command: str) -> str: def add_persist() -> str: if not persist: - return " dbt run-operation drop_branch_schemas" - return " true " + return "dbt run-operation drop_branch_schemas" + return "" def run_test(): From 2f0aca45234431ba1a8a36ddae0a9c133b6418d6 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Wed, 8 Dec 2021 11:25:09 -0500 Subject: [PATCH 09/19] Remove add_persist function just use a condition in make_cmd to prevent appending an empty string when --persist is set --- palm/plugins/dbt/commands/cmd_cycle.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/palm/plugins/dbt/commands/cmd_cycle.py b/palm/plugins/dbt/commands/cmd_cycle.py index f191d2d..8bf04f8 100644 --- a/palm/plugins/dbt/commands/cmd_cycle.py +++ b/palm/plugins/dbt/commands/cmd_cycle.py @@ -35,13 +35,7 @@ def add_select(command: str) -> str: command += " --select " + " ".join(select) return command - def add_persist() -> str: - if not persist: - return "dbt run-operation drop_branch_schemas" - return "" - def run_test(): - return " && ".join( ( add_models("dbt run"), @@ -57,7 +51,9 @@ def make_cmd(): commands.append(add_select(seed_cmd)) commands.append(run_test()) - commands.append(add_persist()) + + if not persist: + commands.append("dbt run-operation drop_branch_schemas") return " && ".join(list(filter(None, commands))) From 6b17d2b0fe971bf5cf8a370883a5c11a6f70d7ec Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Wed, 8 Dec 2021 13:31:49 -0500 Subject: [PATCH 10/19] Hard pin the version of black we are using to lint, ensure version matches between dev and CI --- .github/workflows/pull-request.yaml | 3 ++- dev-requirements.txt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 1ed2ab6..c9c7d97 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -35,4 +35,5 @@ jobs: - uses: actions/checkout@v2 - uses: psf/black@stable with: - options: '--check --diff --skip-string-normalization --exclude="\.tpl\.py"' \ No newline at end of file + options: '--check --diff --skip-string-normalization --exclude="\.tpl\.py"' + version: '21.12b0' \ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt index 0db2ed6..512521a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ -r palm/plugins/dbt/requirements.txt pytest >= 6.2, < 6.3 -black >= 19.10b0, < 20.0 \ No newline at end of file +black == 21.12b0 \ No newline at end of file From 9dc20e61408aa4d34eaa62d984ef8bf691d332a6 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Wed, 8 Dec 2021 16:05:15 -0500 Subject: [PATCH 11/19] Add volume mount for deps directory to docker-compose.yaml template * Includes logic to detect configured changes to the default location for dbt_modules --- palm/plugins/dbt/dbt_containerizer.py | 14 ++++++ .../containerize/docker-compose.yaml | 1 + tests/unit/test_dbt_containerizer.py | 43 +++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/palm/plugins/dbt/dbt_containerizer.py b/palm/plugins/dbt/dbt_containerizer.py index 36928fa..2c698d7 100644 --- a/palm/plugins/dbt/dbt_containerizer.py +++ b/palm/plugins/dbt/dbt_containerizer.py @@ -4,6 +4,7 @@ from palm.containerizer import PythonContainerizer from palm.palm_exceptions import AbortPalm import click +import yaml class DbtContainerizer(PythonContainerizer): @@ -88,6 +89,7 @@ def replacements(self) -> Dict: "project_name": self.project_name, "package_manager": self.package_manager, "dbt_version": self.dbt_version, + "packages_dir": self.get_packages_dir(), } ) return replacements @@ -112,6 +114,18 @@ def write_profile_envs(self) -> None: ) ) + def get_packages_dir(self) -> str: + deps_dir = "dbt_modules" + dbt_confg = self.dbt_project_config() + if dbt_confg.get('modules-path'): + deps_dir = dbt_confg['modules-path'] + if dbt_confg.get('packages-install-path'): + deps_dir = dbt_confg['packages-install-path'] + return deps_dir + + def dbt_project_config(self) -> dict: + return yaml.safe_load(Path("dbt_project.yml").read_text()) + @classmethod def determine_profile_strategy(cls, project_path: "Path") -> Tuple[str, str]: """determines where the on-the-host project diff --git a/palm/plugins/dbt/templates/containerize/docker-compose.yaml b/palm/plugins/dbt/templates/containerize/docker-compose.yaml index 1bf1ea2..8cf2854 100644 --- a/palm/plugins/dbt/templates/containerize/docker-compose.yaml +++ b/palm/plugins/dbt/templates/containerize/docker-compose.yaml @@ -7,6 +7,7 @@ services: dockerfile: Dockerfile volumes: - ./:/app + - /app/{{packages_dir}} {% if profile_volume_mount %} - {{ profile_volume_mount }} {% endif %} diff --git a/tests/unit/test_dbt_containerizer.py b/tests/unit/test_dbt_containerizer.py index 0d06fb6..8b2359d 100644 --- a/tests/unit/test_dbt_containerizer.py +++ b/tests/unit/test_dbt_containerizer.py @@ -1,5 +1,6 @@ import os import pytest +import yaml from unittest import mock from pathlib import Path from palm.plugins.dbt.dbt_containerizer import DbtContainerizer @@ -144,3 +145,45 @@ def test_profile_strategy_none(tmpdir, monkeypatch): None, None, ) + +def test_dbt_project_config(tmpdir, environment): + dbt_config = {"name": 'test_project'} + with open(tmpdir / 'dbt_project.yml', 'w') as f: + f.write(yaml.dump(dbt_config)) + os.chdir(tmpdir) + + templates_dir = ( + Path(__file__).parents[2] / 'palm/plugins/dbt/templates/containerize' + ) + ctx = MockContext(obj=environment) + c = DbtContainerizer(ctx, templates_dir) + + assert c.dbt_project_config() == dbt_config + +def test_dbt_packages_dir(tmpdir, environment): + dbt_config = {"name": 'test_project'} + with open(tmpdir / 'dbt_project.yml', 'w') as f: + f.write(yaml.dump(dbt_config)) + os.chdir(tmpdir) + + templates_dir = ( + Path(__file__).parents[2] / 'palm/plugins/dbt/templates/containerize' + ) + ctx = MockContext(obj=environment) + c = DbtContainerizer(ctx, templates_dir) + + # default value + assert c.get_packages_dir() == 'dbt_modules' + + # modules-path config + dbt_config['modules-path'] = 'custom_modules_path' + with open(tmpdir / 'dbt_project.yml', 'w') as f: + f.write(yaml.dump(dbt_config)) + assert c.get_packages_dir() == 'custom_modules_path' + dbt_config.pop('modules-path') + + # package-install-path config (dbt v1.x) + dbt_config['packages-install-path'] = 'custom_packages_path' + with open(tmpdir / 'dbt_project.yml', 'w') as f: + f.write(yaml.dump(dbt_config)) + assert c.get_packages_dir() == 'custom_packages_path' \ No newline at end of file From a6d9e27a211a50a5109bb1908d7893d3a5e4b1f5 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Wed, 8 Dec 2021 16:06:06 -0500 Subject: [PATCH 12/19] remove cmd_deps - this no longer makes sense, to update deps, the developer should rebuild the image with palm build --- palm/plugins/dbt/commands/cmd_deps.py | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 palm/plugins/dbt/commands/cmd_deps.py diff --git a/palm/plugins/dbt/commands/cmd_deps.py b/palm/plugins/dbt/commands/cmd_deps.py deleted file mode 100644 index 3a49495..0000000 --- a/palm/plugins/dbt/commands/cmd_deps.py +++ /dev/null @@ -1,10 +0,0 @@ -import click -from palm.plugins.dbt.dbt_palm_utils import dbt_env_vars - - -@click.command('deps') -@click.pass_context -def cli(ctx): - """Recompile dbt deps""" - env_vars = dbt_env_vars(ctx.obj.palm.branch) - ctx.obj.run_in_docker('dbt deps', env_vars) From f463b9950015c4cecee875ee445e928f268c3145 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Wed, 8 Dec 2021 16:07:11 -0500 Subject: [PATCH 13/19] Move this section up so the document hierachy makes sense again --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 131fbbe..3218e36 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,11 @@ plugins: - dbt ``` +## Check dbt plugin version + +Check the version of palm-dbt inside a project in which you have configured palm with the dbt plugin: +`palm plugin versions` + ## Palm-ing an existing dbt project palm-dbt ships with a command to containerize and convert your existing dbt project. @@ -33,11 +38,6 @@ For example, if you wanted to containerize your existing dbt project running on palm containerize --version 0.21.0 ``` -## Check dbt plugin version - -Check the version of palm-dbt inside a project in which you have configured palm with the dbt plugin: -`palm plugin versions` - ### Adding palm dbt macros palm-dbt uses the git branch name to set the schema for all commands via env vars. From 3dafe0c84a91cd7b2db8e216b170f7d6b4268e4a Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Wed, 8 Dec 2021 16:20:35 -0500 Subject: [PATCH 14/19] Add documentation around the decisions we made for dbt deps --- README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/README.md b/README.md index 3218e36..a63d10a 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,11 @@ Check the version of palm-dbt inside a project in which you have configured palm `palm plugin versions` ## Palm-ing an existing dbt project + palm-dbt ships with a command to containerize and convert your existing dbt project. For example, if you wanted to containerize your existing dbt project running on 0.21.0, you would run: + ``` palm containerize --version 0.21.0 ``` @@ -98,6 +100,31 @@ Refs will automatically update as well. This way, you can use a single test database and not worry about conflicts between developers, or between branches for the same developer (like during hotfixes). +## palm-dbt and dbt deps + +In palm-dbt we have determined that running `dbt deps` before every command is +problematic for a few reasons: + +1. It takes time, slowing down development, CI, and every production run. +2. If dbt hub or github have an outage, our dbt commands fail and remain broken +until the upstream error is resolved +3. If you forget to run dbt deps, the resulting error messages can be quite confusing. + +To solve these problems, we have decided that running `dbt clean && dbt deps` should +happen in the Dockerfile, when the image is being built. + +To support this decision your project _must_ do the following: + +1. Include `RUN dbt clean && dbt deps` in the Dockerfile +2. Include a `volume` entry in the docker-compose.yaml for the dbt_modules directory +like this `- /app/{{packages_dir}}`, which will prevent the `.:./app` volume from +blowing away the deps generated when the image was built. + +_if you use `palm containerize` this will be done for you!_ + +Additionally, if you need to make changes to your deps you should use `palm build` +to rebuild the image, which will update your deps! + ## Typical palm-dbt workflow From a non-protected branch, running `palm run` will: From efd2e983abeeb7aa33b7f390b7326748bfa720ba Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Wed, 8 Dec 2021 16:23:39 -0500 Subject: [PATCH 15/19] Handle non-existing dbt_project.yml --- palm/plugins/dbt/dbt_containerizer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/palm/plugins/dbt/dbt_containerizer.py b/palm/plugins/dbt/dbt_containerizer.py index 2c698d7..5acaad6 100644 --- a/palm/plugins/dbt/dbt_containerizer.py +++ b/palm/plugins/dbt/dbt_containerizer.py @@ -124,7 +124,10 @@ def get_packages_dir(self) -> str: return deps_dir def dbt_project_config(self) -> dict: - return yaml.safe_load(Path("dbt_project.yml").read_text()) + config_path = Path("dbt_project.yml") + if config_path.exists(): + return yaml.safe_load(config_path.read_text()) + return {} @classmethod def determine_profile_strategy(cls, project_path: "Path") -> Tuple[str, str]: From 99fc6f8d1a3824e3f4fdea426d069f3cc76c3c95 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Wed, 8 Dec 2021 16:25:20 -0500 Subject: [PATCH 16/19] Fix the linter --- palm/plugins/dbt/dbt_containerizer.py | 2 +- tests/unit/test_dbt_containerizer.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/palm/plugins/dbt/dbt_containerizer.py b/palm/plugins/dbt/dbt_containerizer.py index 5acaad6..bee7b1f 100644 --- a/palm/plugins/dbt/dbt_containerizer.py +++ b/palm/plugins/dbt/dbt_containerizer.py @@ -122,7 +122,7 @@ def get_packages_dir(self) -> str: if dbt_confg.get('packages-install-path'): deps_dir = dbt_confg['packages-install-path'] return deps_dir - + def dbt_project_config(self) -> dict: config_path = Path("dbt_project.yml") if config_path.exists(): diff --git a/tests/unit/test_dbt_containerizer.py b/tests/unit/test_dbt_containerizer.py index 8b2359d..5e4b65e 100644 --- a/tests/unit/test_dbt_containerizer.py +++ b/tests/unit/test_dbt_containerizer.py @@ -146,6 +146,7 @@ def test_profile_strategy_none(tmpdir, monkeypatch): None, ) + def test_dbt_project_config(tmpdir, environment): dbt_config = {"name": 'test_project'} with open(tmpdir / 'dbt_project.yml', 'w') as f: @@ -160,12 +161,13 @@ def test_dbt_project_config(tmpdir, environment): assert c.dbt_project_config() == dbt_config + def test_dbt_packages_dir(tmpdir, environment): dbt_config = {"name": 'test_project'} with open(tmpdir / 'dbt_project.yml', 'w') as f: f.write(yaml.dump(dbt_config)) os.chdir(tmpdir) - + templates_dir = ( Path(__file__).parents[2] / 'palm/plugins/dbt/templates/containerize' ) @@ -186,4 +188,4 @@ def test_dbt_packages_dir(tmpdir, environment): dbt_config['packages-install-path'] = 'custom_packages_path' with open(tmpdir / 'dbt_project.yml', 'w') as f: f.write(yaml.dump(dbt_config)) - assert c.get_packages_dir() == 'custom_packages_path' \ No newline at end of file + assert c.get_packages_dir() == 'custom_packages_path' From 62f10a88dadcb9fd43a0c4a19e91e7b133c737f9 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Thu, 9 Dec 2021 10:38:48 -0500 Subject: [PATCH 17/19] Initial support for v1.0 in dbt_containerizer --- palm/plugins/dbt/dbt_containerizer.py | 7 ++++++- tests/unit/test_dbt_containerizer.py | 28 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/palm/plugins/dbt/dbt_containerizer.py b/palm/plugins/dbt/dbt_containerizer.py index bee7b1f..f660393 100644 --- a/palm/plugins/dbt/dbt_containerizer.py +++ b/palm/plugins/dbt/dbt_containerizer.py @@ -67,6 +67,11 @@ def validate_dbt_version(self) -> Tuple[bool, str]: return (True, f'{self.dbt_version} is valid') + @property + def is_dbt_v1(self) -> bool: + """Returns True if dbt version is 1.x.x""" + return self.dbt_version.split(".")[0] == '1' + @property def replacements(self) -> Dict: """ @@ -115,7 +120,7 @@ def write_profile_envs(self) -> None: ) def get_packages_dir(self) -> str: - deps_dir = "dbt_modules" + deps_dir = "dbt_packages" if self.is_dbt_v1 else "dbt_modules" dbt_confg = self.dbt_project_config() if dbt_confg.get('modules-path'): deps_dir = dbt_confg['modules-path'] diff --git a/tests/unit/test_dbt_containerizer.py b/tests/unit/test_dbt_containerizer.py index 5e4b65e..56c0784 100644 --- a/tests/unit/test_dbt_containerizer.py +++ b/tests/unit/test_dbt_containerizer.py @@ -189,3 +189,31 @@ def test_dbt_packages_dir(tmpdir, environment): with open(tmpdir / 'dbt_project.yml', 'w') as f: f.write(yaml.dump(dbt_config)) assert c.get_packages_dir() == 'custom_packages_path' + +# v1.0 support - Note that we are not currently supporting v1.0 but these tests +# are here to document where we are building in support for v1.0 changes + +def test_is_dbt_v1(environment): + ctx = MockContext(obj=environment) + c = DbtContainerizer(ctx, Path('.')) + + # Default is not v1.0 (yet) + assert not c.is_dbt_v1 + + # True if dbt_version is passed as v1.0.x + c = DbtContainerizer(ctx, Path('.'), '1.0.0') + assert c.is_dbt_v1 + +def test_dbt_packages_dir_supports_v1(tmpdir, environment): + dbt_config = {"name": 'test_project'} + with open(tmpdir / 'dbt_project.yml', 'w') as f: + f.write(yaml.dump(dbt_config)) + os.chdir(tmpdir) + + templates_dir = ( + Path(__file__).parents[2] / 'palm/plugins/dbt/templates/containerize' + ) + ctx = MockContext(obj=environment) + # dbt v1 default value + c = DbtContainerizer(ctx, templates_dir, '1.0.0') + assert c.get_packages_dir() == 'dbt_packages' From 67779eea02d4f861ed55186a3b96498ca6dfee7c Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Thu, 9 Dec 2021 10:41:47 -0500 Subject: [PATCH 18/19] Lint for fun and profit (and passing builds) --- tests/unit/test_dbt_containerizer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/test_dbt_containerizer.py b/tests/unit/test_dbt_containerizer.py index 56c0784..4340fcc 100644 --- a/tests/unit/test_dbt_containerizer.py +++ b/tests/unit/test_dbt_containerizer.py @@ -190,9 +190,11 @@ def test_dbt_packages_dir(tmpdir, environment): f.write(yaml.dump(dbt_config)) assert c.get_packages_dir() == 'custom_packages_path' + # v1.0 support - Note that we are not currently supporting v1.0 but these tests # are here to document where we are building in support for v1.0 changes + def test_is_dbt_v1(environment): ctx = MockContext(obj=environment) c = DbtContainerizer(ctx, Path('.')) @@ -204,6 +206,7 @@ def test_is_dbt_v1(environment): c = DbtContainerizer(ctx, Path('.'), '1.0.0') assert c.is_dbt_v1 + def test_dbt_packages_dir_supports_v1(tmpdir, environment): dbt_config = {"name": 'test_project'} with open(tmpdir / 'dbt_project.yml', 'w') as f: From 7b78dc291b50f955211917b2cbf88239d9e564fb Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Wed, 5 Jan 2022 10:29:25 -0500 Subject: [PATCH 19/19] Bump version and add changelog for v0.2 release --- CHANGELOG.md | 14 ++++++++++++++ setup.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af86ad5..184c2b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Palm dbt Changelog +## 0.2.0 + +> 01/05/2022 + +Features: +- **dbt deps**: palm-dbt now assumes dbt deps are installed when the docker image is built. +Projects containerized by palm-dbt will be set up with this functionality, other projects +may need to adjust their Dockerfile to RUN dbt deps and implement the volume mount +in their docker-compose.yaml. See README for full details. + +Developer Improvement +- **pin version of black**: The version of black used to lint this project has been +pinned to ensure the local version matches the version used in CI + ## 0.1.2 Features: - **Containerize**: There is an explicit prompt to enter the dbt version number when containerizing a dbt project. Issue #34 diff --git a/setup.py b/setup.py index 6b21afc..a53f9a2 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( name='palm-dbt', - version='0.1.2', + version='0.2.0', description='dbt extension for Palm CLI', long_description=long_description, long_description_content_type='text/markdown',