Merge pull request #75 from palmetto/release/v0.5.0

Release/v0.5.0

CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.0] - 2023-03-16
+
+### Added
+
+- Automated release to pypi via GH actions
+- Support for containerizing dbt projects with dbt v1+
+
+### Fixed
+
+- Validation of plugin config where prod manifest is not used
+- Default dbt run options do not include stateful selection unless --defer is used.
+- Support for Pyyaml v6 (removed upper version pin)
+
 ## [0.4.0] - 2022-12-09
 
 ### Added
@@ -84,7 +97,6 @@ pinned to ensure the local version matches the version used in CI
 
 - **Bug Fixes**: Removed profiles.yml reference in Dockerfile template. Palm-dbt detects profiles.yml automatically.
 
-
 ## [0.1.1] - 2021-12-02
 
 ### Fixed

palm/plugins/dbt/commands/cmd_containerize.py

@@ -8,12 +8,12 @@
 @click.option(
     "--version",
     multiple=False,
-    help="dbt version to use (e.g. 0.21.0)",
+    help="dbt version to use (e.g. 1.0.1)",
 )
 @click.pass_context
 def cli(ctx, version: Optional[str]):
     if not version:
-        version = click.prompt("Enter dbt version to use", type=str, default="0.21.0")
+        version = click.prompt("Enter dbt version to use", type=str, default="1.0.1")
 
     template_dir = Path(Path(__file__).parents[1], "templates") / "containerize"
     DbtContainerizer(ctx, template_dir, version).run()

palm/plugins/dbt/commands/cmd_run.py

@@ -107,18 +107,13 @@ def build_run_command(
     vars: Optional[str] = None,
 ) -> str:
     cmd = []
-    if full_refresh:
-        full_refresh_option = "--full-refresh"
-    else:
-        full_refresh_option = ""
+    full_refresh_option = " --full-refresh" if full_refresh else ""
 
     if not no_seed:
-        cmd.append(f"dbt seed {full_refresh_option}")
+        cmd.append(f"dbt seed{full_refresh_option}")
         cmd.append("&&")
 
-    cmd.append(f"dbt run {full_refresh_option}")
-    if not targets:
-        cmd.extend(["--select", "state:new", "state:modified+"])
+    cmd.append(f"dbt run{full_refresh_option}")
     if targets:
         cmd.append("--select")
         cmd.extend(targets)
@@ -127,9 +122,9 @@ def build_run_command(
         cmd.extend(exclude)
     if not no_fail_fast:
         cmd.append("--fail-fast")
-    if full_refresh:
-        cmd.append("--full-refresh")
     if defer:
+        if not targets:
+            cmd.extend(["--select", "state:new", "state:modified+"])
         cmd.append("--defer")
     if vars:
         cmd.append(f"--vars '{vars}'")

palm/plugins/dbt/dbt_containerizer.py

@@ -13,7 +13,7 @@ class DbtContainerizer(PythonContainerizer):
     """
 
     def __init__(
-        self, ctx, template_dir: Path, dbt_version: Optional[str] = '0.21.0'
+        self, ctx, template_dir: Path, dbt_version: Optional[str] = '1.0.1'
     ) -> None:
         self.ctx = ctx
         self.project_name = ctx.obj.palm.image_name
@@ -49,7 +49,7 @@ def validate_dbt_version(self) -> Tuple[bool, str]:
         """
         semver = self.dbt_version.split(".")
         minimum_version = ['0', '19']
-        maximum_version = ['0', '21']
+        maximum_version = ['1', '3']
 
         if semver[0] <= minimum_version[0] and semver[1] < minimum_version[1]:
             return (

palm/plugins/dbt/plugin_config.py

@@ -1,12 +1,13 @@
 import click
 
+from typing import Optional
 from pathlib import Path
 from pydantic import BaseModel
 from palm.plugins.base_plugin_config import BasePluginConfig
 
 
 class dbtPluginConfigModel(BaseModel):
-    dbt_artifacts_prod: str
+    dbt_artifacts_prod: Optional[str]
     dbt_artifacts_local: str
 
 

palm/plugins/dbt/requirements.txt

@@ -1,3 +1,3 @@
 palm >=2.5.1, <3.0
-pyyaml >= 5.0, < 5.5
+pyyaml >= 5.0
 sqlparse >= 0.3.1

palm/plugins/dbt/templates/containerize/Dockerfile.txt

@@ -1,4 +1,4 @@
-FROM fishtownanalytics/dbt:{{dbt_version}}
+FROM ghcr.io/dbt-labs/dbt-core:{{dbt_version}}
 COPY . /app/
 WORKDIR /app
 ENV PYTHONPATH=${PYTHONPATH}:${PWD}

setup.py

@@ -21,7 +21,7 @@
 
 setup(
     name='palm-dbt',
-    version='0.4.0',
+    version='0.5.0',
     description='dbt extension for Palm CLI',
     long_description=long_description,
     long_description_content_type='text/markdown',

tests/unit/test_dbt_containerizer.py

@@ -95,18 +95,23 @@ def test_validate_dbt_version(environment):
     is_valid, message = c.validate_dbt_version()
     assert is_valid
 
-    # Maximum version is valid
+    # Supported version is valid
     c = DbtContainerizer(ctx, templates_dir, '0.21.1')
     is_valid, message = c.validate_dbt_version()
     assert is_valid
 
-    # Does not support above maximum version
-    c = DbtContainerizer(ctx, templates_dir, '0.22.0')
+    # Next major is valid
+    c = DbtContainerizer(ctx, templates_dir, '1.0.0')
+    is_valid, message = c.validate_dbt_version()
+    assert is_valid
+
+    # Next minor is invalid
+    c = DbtContainerizer(ctx, templates_dir, '1.4.0')
     is_valid, message = c.validate_dbt_version()
     assert not is_valid
 
     # Next major is invalid
-    c = DbtContainerizer(ctx, templates_dir, '1.0.0')
+    c = DbtContainerizer(ctx, templates_dir, '2.1.0')
     is_valid, message = c.validate_dbt_version()
     assert not is_valid
 
@@ -201,7 +206,10 @@ def test_dbt_packages_dir(tmpdir, environment):
     c = DbtContainerizer(ctx, templates_dir)
 
     # default value
-    assert c.get_packages_dir() == 'dbt_modules'
+    if not c.is_dbt_v1:
+        assert c.get_packages_dir() == 'dbt_modules'
+    else:
+        assert c.get_packages_dir() == 'dbt_packages'
 
     # modules-path config
     dbt_config['modules-path'] = 'custom_modules_path'
@@ -217,17 +225,10 @@ def test_dbt_packages_dir(tmpdir, environment):
     assert c.get_packages_dir() == 'custom_packages_path'
 
 
-# v1.0 support - Note that we are not currently supporting v1.0 but these tests
-# are here to document where we are building in support for v1.0 changes
-
-
 def test_is_dbt_v1(environment):
     ctx = MockContext(obj=environment)
     c = DbtContainerizer(ctx, Path('.'))
 
-    # Default is not v1.0 (yet)
-    assert not c.is_dbt_v1
-
     # True if dbt_version is passed as v1.0.x
     c = DbtContainerizer(ctx, Path('.'), '1.0.0')
     assert c.is_dbt_v1