diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c465ca..287b9f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.5.0] - 2023-03-16 + +### Added + +- Automated release to pypi via GH actions +- Support for containerizing dbt projects with dbt v1+ + +### Fixed + +- Validation of plugin config where prod manifest is not used +- Default dbt run options do not include stateful selection unless --defer is used. +- Support for Pyyaml v6 (removed upper version pin) + ## [0.4.0] - 2022-12-09 ### Added @@ -84,7 +97,6 @@ pinned to ensure the local version matches the version used in CI - **Bug Fixes**: Removed profiles.yml reference in Dockerfile template. Palm-dbt detects profiles.yml automatically. - ## [0.1.1] - 2021-12-02 ### Fixed diff --git a/palm/plugins/dbt/commands/cmd_containerize.py b/palm/plugins/dbt/commands/cmd_containerize.py index 3326828..8ac3f33 100644 --- a/palm/plugins/dbt/commands/cmd_containerize.py +++ b/palm/plugins/dbt/commands/cmd_containerize.py @@ -8,12 +8,12 @@ @click.option( "--version", multiple=False, - help="dbt version to use (e.g. 0.21.0)", + help="dbt version to use (e.g. 1.0.1)", ) @click.pass_context def cli(ctx, version: Optional[str]): if not version: - version = click.prompt("Enter dbt version to use", type=str, default="0.21.0") + version = click.prompt("Enter dbt version to use", type=str, default="1.0.1") template_dir = Path(Path(__file__).parents[1], "templates") / "containerize" DbtContainerizer(ctx, template_dir, version).run() diff --git a/palm/plugins/dbt/commands/cmd_run.py b/palm/plugins/dbt/commands/cmd_run.py index 5ac34ed..38d06e1 100644 --- a/palm/plugins/dbt/commands/cmd_run.py +++ b/palm/plugins/dbt/commands/cmd_run.py @@ -107,18 +107,13 @@ def build_run_command( vars: Optional[str] = None, ) -> str: cmd = [] - if full_refresh: - full_refresh_option = "--full-refresh" - else: - full_refresh_option = "" + full_refresh_option = " --full-refresh" if full_refresh else "" if not no_seed: - cmd.append(f"dbt seed {full_refresh_option}") + cmd.append(f"dbt seed{full_refresh_option}") cmd.append("&&") - cmd.append(f"dbt run {full_refresh_option}") - if not targets: - cmd.extend(["--select", "state:new", "state:modified+"]) + cmd.append(f"dbt run{full_refresh_option}") if targets: cmd.append("--select") cmd.extend(targets) @@ -127,9 +122,9 @@ def build_run_command( cmd.extend(exclude) if not no_fail_fast: cmd.append("--fail-fast") - if full_refresh: - cmd.append("--full-refresh") if defer: + if not targets: + cmd.extend(["--select", "state:new", "state:modified+"]) cmd.append("--defer") if vars: cmd.append(f"--vars '{vars}'") diff --git a/palm/plugins/dbt/dbt_containerizer.py b/palm/plugins/dbt/dbt_containerizer.py index f660393..e57aef8 100644 --- a/palm/plugins/dbt/dbt_containerizer.py +++ b/palm/plugins/dbt/dbt_containerizer.py @@ -13,7 +13,7 @@ class DbtContainerizer(PythonContainerizer): """ def __init__( - self, ctx, template_dir: Path, dbt_version: Optional[str] = '0.21.0' + self, ctx, template_dir: Path, dbt_version: Optional[str] = '1.0.1' ) -> None: self.ctx = ctx self.project_name = ctx.obj.palm.image_name @@ -49,7 +49,7 @@ def validate_dbt_version(self) -> Tuple[bool, str]: """ semver = self.dbt_version.split(".") minimum_version = ['0', '19'] - maximum_version = ['0', '21'] + maximum_version = ['1', '3'] if semver[0] <= minimum_version[0] and semver[1] < minimum_version[1]: return ( diff --git a/palm/plugins/dbt/plugin_config.py b/palm/plugins/dbt/plugin_config.py index a3869a7..cf03043 100644 --- a/palm/plugins/dbt/plugin_config.py +++ b/palm/plugins/dbt/plugin_config.py @@ -1,12 +1,13 @@ import click +from typing import Optional from pathlib import Path from pydantic import BaseModel from palm.plugins.base_plugin_config import BasePluginConfig class dbtPluginConfigModel(BaseModel): - dbt_artifacts_prod: str + dbt_artifacts_prod: Optional[str] dbt_artifacts_local: str diff --git a/palm/plugins/dbt/requirements.txt b/palm/plugins/dbt/requirements.txt index 9bc3026..e6871e9 100644 --- a/palm/plugins/dbt/requirements.txt +++ b/palm/plugins/dbt/requirements.txt @@ -1,3 +1,3 @@ palm >=2.5.1, <3.0 -pyyaml >= 5.0, < 5.5 +pyyaml >= 5.0 sqlparse >= 0.3.1 diff --git a/palm/plugins/dbt/templates/containerize/Dockerfile.txt b/palm/plugins/dbt/templates/containerize/Dockerfile.txt index 6cf3d2a..e9d249a 100644 --- a/palm/plugins/dbt/templates/containerize/Dockerfile.txt +++ b/palm/plugins/dbt/templates/containerize/Dockerfile.txt @@ -1,4 +1,4 @@ -FROM fishtownanalytics/dbt:{{dbt_version}} +FROM ghcr.io/dbt-labs/dbt-core:{{dbt_version}} COPY . /app/ WORKDIR /app ENV PYTHONPATH=${PYTHONPATH}:${PWD} diff --git a/setup.py b/setup.py index e46f5c4..e0f7e6f 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( name='palm-dbt', - version='0.4.0', + version='0.5.0', description='dbt extension for Palm CLI', long_description=long_description, long_description_content_type='text/markdown', diff --git a/tests/unit/test_dbt_containerizer.py b/tests/unit/test_dbt_containerizer.py index 5e1045d..80808cd 100644 --- a/tests/unit/test_dbt_containerizer.py +++ b/tests/unit/test_dbt_containerizer.py @@ -95,18 +95,23 @@ def test_validate_dbt_version(environment): is_valid, message = c.validate_dbt_version() assert is_valid - # Maximum version is valid + # Supported version is valid c = DbtContainerizer(ctx, templates_dir, '0.21.1') is_valid, message = c.validate_dbt_version() assert is_valid - # Does not support above maximum version - c = DbtContainerizer(ctx, templates_dir, '0.22.0') + # Next major is valid + c = DbtContainerizer(ctx, templates_dir, '1.0.0') + is_valid, message = c.validate_dbt_version() + assert is_valid + + # Next minor is invalid + c = DbtContainerizer(ctx, templates_dir, '1.4.0') is_valid, message = c.validate_dbt_version() assert not is_valid # Next major is invalid - c = DbtContainerizer(ctx, templates_dir, '1.0.0') + c = DbtContainerizer(ctx, templates_dir, '2.1.0') is_valid, message = c.validate_dbt_version() assert not is_valid @@ -201,7 +206,10 @@ def test_dbt_packages_dir(tmpdir, environment): c = DbtContainerizer(ctx, templates_dir) # default value - assert c.get_packages_dir() == 'dbt_modules' + if not c.is_dbt_v1: + assert c.get_packages_dir() == 'dbt_modules' + else: + assert c.get_packages_dir() == 'dbt_packages' # modules-path config dbt_config['modules-path'] = 'custom_modules_path' @@ -217,17 +225,10 @@ def test_dbt_packages_dir(tmpdir, environment): assert c.get_packages_dir() == 'custom_packages_path' -# v1.0 support - Note that we are not currently supporting v1.0 but these tests -# are here to document where we are building in support for v1.0 changes - - def test_is_dbt_v1(environment): ctx = MockContext(obj=environment) c = DbtContainerizer(ctx, Path('.')) - # Default is not v1.0 (yet) - assert not c.is_dbt_v1 - # True if dbt_version is passed as v1.0.x c = DbtContainerizer(ctx, Path('.'), '1.0.0') assert c.is_dbt_v1