Merge pull request #47 from palmetto/release/v0.2.0

Release/v0.2.0

.github/workflows/pull-request.yaml

@@ -35,4 +35,5 @@ jobs:
       - uses: actions/checkout@v2
       - uses: psf/black@stable
         with:
-          options: '--check --diff --skip-string-normalization --exclude="\.tpl\.py"'
+          options: '--check --diff --skip-string-normalization --exclude="\.tpl\.py"'
+          version: '21.12b0'

CHANGELOG.md

@@ -1,5 +1,19 @@
 # Palm dbt Changelog
 
+## 0.2.0
+
+> 01/05/2022
+
+Features:
+- **dbt deps**: palm-dbt now assumes dbt deps are installed when the docker image is built.
+Projects containerized by palm-dbt will be set up with this functionality, other projects
+may need to adjust their Dockerfile to RUN dbt deps and implement the volume mount
+in their docker-compose.yaml. See README for full details.
+
+Developer Improvement
+- **pin version of black**: The version of black used to lint this project has been
+pinned to ensure the local version matches the version used in CI
+
 ## 0.1.2
 Features:
 - **Containerize**: There is an explicit prompt to enter the dbt version number when containerizing a dbt project. Issue #34

README.md

@@ -25,19 +25,21 @@ plugins:
   - dbt
 ```
 
+## Check dbt plugin version
+
+Check the version of palm-dbt inside a project in which you have configured palm with the dbt plugin:
+`palm plugin versions`
+
 ## Palm-ing an existing dbt project
+
 palm-dbt ships with a command to containerize and convert your existing dbt project.
 
 For example, if you wanted to containerize your existing dbt project running on 0.21.0, you would run:
+
 ```
   palm containerize --version 0.21.0
 ```
 
-## Check dbt plugin version
-
-Check the version of palm-dbt inside a project in which you have configured palm with the dbt plugin:
-`palm plugin versions`
-
 ### Adding palm dbt macros
 
 palm-dbt uses the git branch name to set the schema for all commands via env vars.
@@ -98,12 +100,37 @@ Refs will automatically update as well. This way, you can use a single test
 database and not worry about conflicts between developers, or between branches 
 for the same developer (like during hotfixes). 
 
+## palm-dbt and dbt deps
+
+In palm-dbt we have determined that running `dbt deps` before every command is
+problematic for a few reasons:
+
+1. It takes time, slowing down development, CI, and every production run.
+2. If dbt hub or github have an outage, our dbt commands fail and remain broken
+until the upstream error is resolved
+3. If you forget to run dbt deps, the resulting error messages can be quite confusing.
+
+To solve these problems, we have decided that running `dbt clean && dbt deps` should
+happen in the Dockerfile, when the image is being built.
+
+To support this decision your project _must_ do the following:
+
+1. Include `RUN dbt clean && dbt deps` in the Dockerfile
+2. Include a `volume` entry in the docker-compose.yaml for the dbt_modules directory
+like this `- /app/{{packages_dir}}`, which will prevent the `.:./app` volume from
+blowing away the deps generated when the image was built.
+
+_if you use `palm containerize` this will be done for you!_
+
+Additionally, if you need to make changes to your deps you should use `palm build`
+to rebuild the image, which will update your deps!
+
 ## Typical palm-dbt workflow
 
 From a non-protected branch, running `palm run` will:
 1. drop (if it exists) the namespaced schema in development
 2. create the namespaced schema in development
-3. clean, deps, seed and run
+3. seed and run
 4. drop the namespaced schema in development
 
 Why drop it? so your testing is atomic. 

dev-requirements.txt

@@ -1,3 +1,3 @@
 -r palm/plugins/dbt/requirements.txt
 pytest >= 6.2, < 6.3
-black >= 19.10b0, < 20.0
+black == 21.12b0

palm/plugins/dbt/commands/cmd_cleanup.py

@@ -9,7 +9,7 @@
 def cli(ctx):
     """Removes any artifacts from Snowflake related to the current branch."""
 
-    cmd = "dbt clean && dbt deps && dbt run-operation drop_branch_schemas"
+    cmd = "dbt run-operation drop_branch_schemas && dbt clean && dbt deps"
     env_vars = dbt_env_vars(ctx.obj.palm.branch)
     success, msg = ctx.obj.run_in_docker(cmd, env_vars)
     click.secho(msg, fg="green" if success else "red")

palm/plugins/dbt/commands/cmd_compile.py

@@ -5,17 +5,13 @@
 
 @click.command("compile")
 @click.option("--models", multiple=True, help="see dbt docs on models flag")
-@click.option("--fast", is_flag=True, help="will skip clean/deps/seed")
 @click.pass_context
-def cli(ctx, fast: bool, models: Optional[Tuple] = tuple()):
+def cli(ctx, models: Optional[Tuple] = tuple()):
     """Cleans up target directory and dependencies, then compiles dbt"""
 
-    if fast:
-        cmd = "dbt compile"
-    else:
-        cmd = "dbt clean && dbt deps && dbt compile"
+    cmd = "dbt compile"
     if models:
-        cmd += f" --models " + " ".join(models)
+        cmd += f" --models {' '.join(models)}"
 
     env_vars = dbt_env_vars(ctx.obj.palm.branch)
     success, msg = ctx.obj.run_in_docker(cmd, env_vars)

palm/plugins/dbt/commands/cmd_cycle.py

@@ -8,7 +8,6 @@
 
 @click.command("cycle")
 @click.argument("count", type=int, default=2)
-@click.option("--fast", is_flag=True, help="will skip clean/deps/seed")
 @click.option(
     "--persist", is_flag=True, help="will not drop the test schema at the end"
 )
@@ -19,7 +18,6 @@
 def cli(
     ctx,
     count: int,
-    fast: bool,
     persist: bool,
     no_seed: bool,
     models: Optional[Tuple] = tuple(),
@@ -37,13 +35,7 @@ def add_select(command: str) -> str:
             command += " --select " + " ".join(select)
         return command
 
-    def add_persist() -> str:
-        if not persist:
-            return " dbt run-operation drop_branch_schemas"
-        return " true "
-
     def run_test():
-
         return " && ".join(
             (
                 add_models("dbt run"),
@@ -53,22 +45,16 @@ def run_test():
         )
 
     def make_cmd():
-        if fast:
-            if no_seed:
-                seed_cmd = ""
-            else:
-                seed_cmd = "dbt seed --full-refresh"
-            commands = [add_select(seed_cmd), run_test(), add_persist()]
-        else:
-            if no_seed:
-                seed_cmd = ""
-            else:
-                seed_cmd = " && dbt seed --full-refresh "
-            commands = [
-                add_select("dbt clean && dbt deps" + seed_cmd),
-                run_test(),
-                add_persist(),
-            ]
+        commands = []
+        if not no_seed:
+            seed_cmd = "dbt seed --full-refresh"
+            commands.append(add_select(seed_cmd))
+
+        commands.append(run_test())
+
+        if not persist:
+            commands.append("dbt run-operation drop_branch_schemas")
+
         return " && ".join(list(filter(None, commands)))
 
     env_vars = dbt_env_vars(ctx.obj.palm.branch)

palm/plugins/dbt/commands/cmd_run.py

@@ -4,7 +4,6 @@
 
 
 @click.command("run")
-@click.option("--fast", is_flag=True, help="will skip clean/deps/seed")
 @click.option(
     "--no-fail-fast",
     is_flag=True,
@@ -27,7 +26,6 @@
 @click.pass_context
 def cli(
     ctx,
-    fast: bool,
     no_fail_fast: bool,
     persist: bool,
     full_refresh: bool,

palm/plugins/dbt/commands/cmd_snapshot.py

@@ -8,15 +8,11 @@
     "--persist", is_flag=True, help="will not drop the test schema at the end"
 )
 @click.option("--select", multiple=True)
-@click.option("--fast", is_flag=True, help="will skip clean/deps/seed")
 @click.pass_context
-def cli(ctx, persist: bool, fast: bool, select: Optional[Tuple] = tuple()):
+def cli(ctx, persist: bool, select: Optional[Tuple] = tuple()):
     """Executes the DBT snapshots."""
 
-    if fast:
-        cmd = "dbt snapshot"
-    else:
-        cmd = "dbt clean && dbt deps && dbt snapshot"
+    cmd = "dbt snapshot"
     if select:
         cmd += f" --select " + " ".join(select)
     if not persist:

palm/plugins/dbt/commands/cmd_test.py

@@ -4,7 +4,6 @@
 
 
 @click.command('test')
-@click.option("--fast", is_flag=True, help="will skip clean/deps/seed")
 @click.option(
     "--persist", is_flag=True, help="will not drop the test schema at the end"
 )
@@ -15,7 +14,6 @@
 @click.pass_context
 def cli(
     ctx,
-    fast: bool,
     persist: bool,
     no_seed: bool,
     no_fail_fast: bool,

palm/plugins/dbt/dbt_containerizer.py

@@ -4,6 +4,7 @@
 from palm.containerizer import PythonContainerizer
 from palm.palm_exceptions import AbortPalm
 import click
+import yaml
 
 
 class DbtContainerizer(PythonContainerizer):
@@ -66,6 +67,11 @@ def validate_dbt_version(self) -> Tuple[bool, str]:
 
         return (True, f'{self.dbt_version} is valid')
 
+    @property
+    def is_dbt_v1(self) -> bool:
+        """Returns True if dbt version is 1.x.x"""
+        return self.dbt_version.split(".")[0] == '1'
+
     @property
     def replacements(self) -> Dict:
         """
@@ -88,6 +94,7 @@ def replacements(self) -> Dict:
                 "project_name": self.project_name,
                 "package_manager": self.package_manager,
                 "dbt_version": self.dbt_version,
+                "packages_dir": self.get_packages_dir(),
             }
         )
         return replacements
@@ -112,6 +119,21 @@ def write_profile_envs(self) -> None:
                 )
             )
 
+    def get_packages_dir(self) -> str:
+        deps_dir = "dbt_packages" if self.is_dbt_v1 else "dbt_modules"
+        dbt_confg = self.dbt_project_config()
+        if dbt_confg.get('modules-path'):
+            deps_dir = dbt_confg['modules-path']
+        if dbt_confg.get('packages-install-path'):
+            deps_dir = dbt_confg['packages-install-path']
+        return deps_dir
+
+    def dbt_project_config(self) -> dict:
+        config_path = Path("dbt_project.yml")
+        if config_path.exists():
+            return yaml.safe_load(config_path.read_text())
+        return {}
+
     @classmethod
     def determine_profile_strategy(cls, project_path: "Path") -> Tuple[str, str]:
         """determines where the on-the-host project

palm/plugins/dbt/dbt_palm_utils.py

@@ -9,45 +9,10 @@ def shell_options(command_name: str, **kwargs):
     # pop off unused kwargs caused by use of importlib in palm
     kwargs.pop("ctx")
 
-    if kwargs["fast"]:
-        for k in ("fast", "select"):
-            kwargs.pop(k, None)
-        return _short_cycle(command_name, **kwargs)
-        return
-    kwargs.pop("fast")
-    return _long_cycle(command_name, **kwargs)
+    return _cycle(command_name, **kwargs)
 
 
-def _short_cycle(
-    cmd: str,
-    persist: Optional[bool] = False,
-    no_fail_fast: Optional[bool] = False,
-    full_refresh: Optional[bool] = False,
-    no_seed: Optional[bool] = False,
-    models: Optional[Tuple] = (),
-    macros: Optional[Tuple] = (),
-) -> str:
-
-    command = "" if no_seed else f" dbt seed --full-refresh && "
-
-    if macros:
-        command += f"dbt run-operation {macros}"
-    else:
-        command += f"dbt {cmd}"
-        if models:
-            command += " --models " + " ".join(models)
-        if full_refresh:
-            command += " --full-refresh"
-        if not no_fail_fast:
-            command += " --fail-fast"
-
-    if not persist:
-        command += " && dbt run-operation drop_branch_schemas"
-
-    return command
-
-
-def _long_cycle(
+def _cycle(
     cmd: str,
     persist: Optional[bool] = False,
     no_fail_fast: Optional[bool] = False,
@@ -57,27 +22,28 @@ def _long_cycle(
     models: Optional[Tuple] = (),
     macros: Optional[Tuple] = (),
 ) -> str:
-    seed_cmd = "" if no_seed else "&& dbt seed --full-refresh"
-    command = f"dbt clean && dbt deps {seed_cmd}"
+    command = []
+    if not no_seed:
+        command.append("dbt seed --full-refresh")
 
     if macros:
-        run_operation = " && dbt run-operation "
-        command += run_operation + run_operation.join(macros)
+        command.append(f'dbt run-operation + {"&& dbt run-operation".join(macros)}')
     else:
-        command += f" && dbt {cmd}"
+        subcommand = f"dbt {cmd}"
         if select and not no_seed:
-            command += " --select " + " ".join(select)
+            subcommand += " --select " + " ".join(select)
         if models:
-            command += " --models " + " ".join(models)
+            subcommand += " --models " + " ".join(models)
         if full_refresh:
-            command += " --full-refresh"
+            subcommand += " --full-refresh"
         if not no_fail_fast:
-            command += " --fail-fast"
+            subcommand += " --fail-fast"
+        command.append(subcommand)
 
     if not persist:
-        command += " && dbt run-operation drop_branch_schemas"
+        command.append("dbt run-operation drop_branch_schemas")
 
-    return command
+    return ' && '.join(command)
 
 
 def dbt_env_vars(branch: str) -> Dict:

palm/plugins/dbt/templates/containerize/Dockerfile.txt

@@ -4,4 +4,6 @@ WORKDIR /app
 ENV PYTHONPATH=${PYTHONPATH}:${PWD}
 
 RUN ./scripts/entrypoint.sh
+RUN dbt clean && dbt deps
+
 ENTRYPOINT [ "" ]

palm/plugins/dbt/templates/containerize/docker-compose.yaml

@@ -7,6 +7,7 @@ services:
       dockerfile: Dockerfile
     volumes:
       - ./:/app
+      - /app/{{packages_dir}}
     {% if profile_volume_mount %}
       - {{ profile_volume_mount }}
     {% endif %}