From ac22513e0b1e7df18d05dfe4fb29a9e0c415b3c0 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Fri, 30 Jun 2023 16:09:15 -0400 Subject: [PATCH 01/11] Update codeowners to use GH team --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b5d8e98..7f97fde 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,4 @@ # see https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners for configs # review PRs -* @jakeberesford-palmetto @murphymoulds-palmetto @mariahjrogers @evandegler-palmetto @omoorepalmetto +* @palmetto/data-analytics From b139023db6f08dd9b2a4e69508e6e0074247b2de Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Fri, 30 Jun 2023 16:10:10 -0400 Subject: [PATCH 02/11] mariahjrogers can still be a codeowner since she is actively contributing and has write access. --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7f97fde..e1b69fe 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,4 @@ # see https://docs.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners for configs # review PRs -* @palmetto/data-analytics +* @palmetto/data-analytics @mariahjrogers From 0042b7513a12cab4518cd2a7cb258b56c612c097 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Tue, 18 Jul 2023 16:56:34 -0400 Subject: [PATCH 03/11] Add support for yaml selectors to relevant commands Not having this has started to be a thorn in my side --- palm/plugins/dbt/commands/cmd_compile.py | 5 +++++ palm/plugins/dbt/commands/cmd_run.py | 7 +++++++ palm/plugins/dbt/commands/cmd_seed.py | 5 +++++ palm/plugins/dbt/commands/cmd_test.py | 7 +++++++ 4 files changed, 24 insertions(+) diff --git a/palm/plugins/dbt/commands/cmd_compile.py b/palm/plugins/dbt/commands/cmd_compile.py index 54d9a69..0aebbb9 100644 --- a/palm/plugins/dbt/commands/cmd_compile.py +++ b/palm/plugins/dbt/commands/cmd_compile.py @@ -6,12 +6,14 @@ @click.command("compile") @click.option("--models", "-m", multiple=True, help="See dbt docs on models flag") @click.option("--select", "-s", multiple=True, help="See dbt docs on select flag") +@click.option("--selector", multiple=True, help="See dbt docs on selector flag") @click.option("--exclude", "-e", multiple=True, help="See dbt docs on exclude flag") @click.pass_obj def cli( environment, models: Optional[Tuple] = tuple(), select: Optional[Tuple] = tuple(), + selector: Optional[Tuple] = tuple(), exclude: Optional[Tuple] = tuple(), ): """Compiles the dbt repo""" @@ -24,6 +26,9 @@ def cli( if targets: cmd.append("--select") cmd.extend(targets) + if selector: + cmd.append("--selector") + cmd.extend(selector) if exclude: cmd.append('--exclude') cmd.extend(exclude) diff --git a/palm/plugins/dbt/commands/cmd_run.py b/palm/plugins/dbt/commands/cmd_run.py index ffef0ac..5818a8d 100644 --- a/palm/plugins/dbt/commands/cmd_run.py +++ b/palm/plugins/dbt/commands/cmd_run.py @@ -14,6 +14,7 @@ @click.option("--clean", is_flag=True, help="Drop the test schema after the run") @click.option("--models", "-m", multiple=True, help="See dbt docs on models flag") @click.option("--select", "-s", multiple=True, help="See dbt docs on select flag") +@click.option("--selector", multiple=True, help="See dbt docs on selector flag") @click.option("--exclude", "-e", multiple=True, help="See dbt docs on exclude flag") @click.option("--defer", "-d", is_flag=True, help="See dbt docs on defer flag") @click.option("--iterative", "-i", is_flag=True, help="Iterative stateful dbt run") @@ -35,6 +36,7 @@ def cli( seed: bool, models: Optional[Tuple] = tuple(), select: Optional[Tuple] = tuple(), + selector: Optional[Tuple] = tuple(), exclude: Optional[Tuple] = tuple(), vars: Optional[str] = None, ): @@ -67,6 +69,7 @@ def cli( seed=seed, no_fail_fast=(no_fail_fast or iterative), targets=targets, + selector=selector, exclude=exclude, defer=defer, vars=vars, @@ -104,6 +107,7 @@ def build_run_command( seed: bool = True, no_fail_fast: bool = False, targets: Optional[list] = None, + selector: Optional[Tuple] = None, exclude: Optional[Tuple] = None, defer: bool = False, vars: Optional[str] = None, @@ -119,6 +123,9 @@ def build_run_command( if targets: cmd.append("--select") cmd.extend(targets) + if selector: + cmd.append("--selector") + cmd.extend(selector) if exclude: cmd.append("--exclude") cmd.extend(exclude) diff --git a/palm/plugins/dbt/commands/cmd_seed.py b/palm/plugins/dbt/commands/cmd_seed.py index a9bddd5..7b21570 100644 --- a/palm/plugins/dbt/commands/cmd_seed.py +++ b/palm/plugins/dbt/commands/cmd_seed.py @@ -8,6 +8,7 @@ "--clean", is_flag=True, help="drop the test schema after the run is complete" ) @click.option("--select", '-s', multiple=True, help="see dbt docs on select flag") +@click.option("--selector", multiple=True, help="see dbt docs on selector flag") @click.option("--exclude", '-e', multiple=True, help="see dbt docs on exclude flag") @click.option( "--no-full-refresh", @@ -21,6 +22,7 @@ def cli( clean: bool, no_full_refresh: bool, select: Optional[Tuple] = tuple(), + selector: Optional[Tuple] = tuple(), exclude: Optional[Tuple] = tuple(), ): """Run dbt seeds""" @@ -29,6 +31,9 @@ def cli( if select: cmd.append('--select') cmd.extend(select) + if selector: + cmd.append('--selector') + cmd.extend(selector) if exclude: cmd.append('--exclude') cmd.extend(exclude) diff --git a/palm/plugins/dbt/commands/cmd_test.py b/palm/plugins/dbt/commands/cmd_test.py index 0f2bd69..ce03789 100644 --- a/palm/plugins/dbt/commands/cmd_test.py +++ b/palm/plugins/dbt/commands/cmd_test.py @@ -10,6 +10,7 @@ ) @click.option("--models", "-m", multiple=True, help="See dbt docs on models flag") @click.option("--select", "-s", multiple=True, help="See dbt docs on select flag") +@click.option("--selector", multiple=True, help="See dbt docs on selector flag") @click.option("--exclude", "-e", multiple=True, help="See dbt docs on exclude flag") @click.option("--defer", "-d", is_flag=True, help="See dbt docs on defer flag") @click.option( @@ -23,6 +24,7 @@ def cli( defer: bool, models: Optional[Tuple] = tuple(), select: Optional[Tuple] = tuple(), + selector: Optional[Tuple] = tuple(), exclude: Optional[Tuple] = tuple(), ): """Tests the dbt repo""" @@ -50,6 +52,7 @@ def cli( run_cmd = build_test_command( no_fail_fast=no_fail_fast, targets=targets, + selector=selector, exclude=exclude, defer=defer, ) @@ -67,6 +70,7 @@ def build_test_command( defer: bool = False, no_fail_fast: bool = False, targets: Optional[list] = None, + selector: Optional[Tuple] = None, exclude: Optional[Tuple] = None, ) -> str: cmd = [] @@ -75,6 +79,9 @@ def build_test_command( if targets: cmd.append('--select') cmd.extend(targets) + if selector: + cmd.append('--selector') + cmd.extend(selector) if exclude: cmd.append('--exclude') cmd.extend(exclude) From 01826d0a629719ee80d86f6d38b8c1dfc4fec2a2 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Fri, 8 Sep 2023 15:27:07 -0400 Subject: [PATCH 04/11] Add dbt version detection dbt version will be stored in .palm/config.yml This is a required configuration and will need to be updated by maintainers of the host project. Running `palm plugin configure --name dbt` will do automatic detection. --- palm/plugins/dbt/dbt_version_detection.py | 25 +++++++++++++++++++++++ palm/plugins/dbt/plugin_config.py | 16 +++++++++++++++ palm/plugins/dbt/requirements.txt | 1 + 3 files changed, 42 insertions(+) create mode 100644 palm/plugins/dbt/dbt_version_detection.py diff --git a/palm/plugins/dbt/dbt_version_detection.py b/palm/plugins/dbt/dbt_version_detection.py new file mode 100644 index 0000000..f581a71 --- /dev/null +++ b/palm/plugins/dbt/dbt_version_detection.py @@ -0,0 +1,25 @@ +from email.mime import image +from pathlib import Path +import yaml + +from palm.utils import run_in_docker + +def get_dbt_version(): + image_name = _get_image_name() + cmd = "pip list | grep dbt-core" + success, msg = run_in_docker(cmd, image_name, capture_output=True) + if not success: + raise ValueError(f"Error getting dbt version: {msg}") + return msg.strip().split(" ")[-1] + + +def _get_image_name(): + """Get the image name from the .palm/config.yaml file. + + This isn't great, but we don't want to instantiate a PalmConfig object here. + """ + palm_config_path = Path(".palm/config.yaml") + if not palm_config_path.exists(): + raise ValueError("No .palm/config.yaml file found") + palm_config = yaml.safe_load(palm_config_path.read_text()) + return palm_config["image_name"] diff --git a/palm/plugins/dbt/plugin_config.py b/palm/plugins/dbt/plugin_config.py index cf03043..aa90591 100644 --- a/palm/plugins/dbt/plugin_config.py +++ b/palm/plugins/dbt/plugin_config.py @@ -2,13 +2,27 @@ from typing import Optional from pathlib import Path +import semver from pydantic import BaseModel from palm.plugins.base_plugin_config import BasePluginConfig +from palm.plugins.dbt.dbt_version_detection import get_dbt_version class dbtPluginConfigModel(BaseModel): dbt_artifacts_prod: Optional[str] dbt_artifacts_local: str + dbt_version: str + + def dbt_version_semver(self) -> str: + return semver.Version.parse(self.dbt_version) + + def is_dbt_version_greater_than(self, version: str) -> bool: + target_version = semver.Version.parse(version) + return self.dbt_version_semver() > target_version + + def is_dbt_version_less_than(self, version: str) -> bool: + target_version = semver.Version.parse(version) + return self.dbt_version_semver() < target_version class DbtPluginConfig(BasePluginConfig): @@ -35,4 +49,6 @@ def set(self) -> dict: ) config['dbt_artifacts_local'] = str(local) + config['dbt_version'] = get_dbt_version() + return config diff --git a/palm/plugins/dbt/requirements.txt b/palm/plugins/dbt/requirements.txt index e6871e9..d37e902 100644 --- a/palm/plugins/dbt/requirements.txt +++ b/palm/plugins/dbt/requirements.txt @@ -1,3 +1,4 @@ palm >=2.5.1, <3.0 pyyaml >= 5.0 sqlparse >= 0.3.1 +semver >= 3.0.1 From ad841c1ba52ee1f65a9a7570ba761ed837af7c64 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Fri, 8 Sep 2023 15:28:05 -0400 Subject: [PATCH 05/11] Add conditional ENV var names based on configured dbt version Projects running >= 1.5 will not see deprecation warnings --- palm/plugins/dbt/commands/cmd_run.py | 15 ++++++++++++--- palm/plugins/dbt/commands/cmd_test.py | 15 ++++++++++++--- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/palm/plugins/dbt/commands/cmd_run.py b/palm/plugins/dbt/commands/cmd_run.py index ffef0ac..5d3bae2 100644 --- a/palm/plugins/dbt/commands/cmd_run.py +++ b/palm/plugins/dbt/commands/cmd_run.py @@ -137,10 +137,19 @@ def build_run_command( def set_env_vars(environment, stateful: bool, defer: bool = False) -> dict: plugin_config = environment.plugin_config('dbt') env_vars = dbt_env_vars(environment.palm.branch) + + # These env vars are renamed in dbt v1.5.0, old env vars are deprecated + if plugin_config.is_dbt_version_greater_than("1.5.0"): + defer_env_var = "DBT_DEFER" + state_env_var = "DBT_STATE" + else: + defer_env_var = "DBT_DEFER_TO_STATE" + state_env_var = "DBT_ARTIFACT_STATE_PATH" + if stateful: - env_vars['DBT_DEFER_TO_STATE'] = 'true' + env_vars[defer_env_var] = 'true' if defer: - env_vars['DBT_ARTIFACT_STATE_PATH'] = plugin_config.dbt_artifacts_prod + env_vars[state_env_var] = plugin_config.dbt_artifacts_prod else: - env_vars['DBT_ARTIFACT_STATE_PATH'] = plugin_config.dbt_artifacts_local + env_vars[state_env_var] = plugin_config.dbt_artifacts_local return env_vars diff --git a/palm/plugins/dbt/commands/cmd_test.py b/palm/plugins/dbt/commands/cmd_test.py index 0f2bd69..d0f7414 100644 --- a/palm/plugins/dbt/commands/cmd_test.py +++ b/palm/plugins/dbt/commands/cmd_test.py @@ -91,9 +91,18 @@ def build_test_command( def set_env_vars(environment, defer: bool = False) -> dict: plugin_config = environment.plugin_config('dbt') env_vars = dbt_env_vars(environment.palm.branch) + + # These env vars are renamed in dbt v1.5.0, old env vars are deprecated + if plugin_config.is_dbt_version_greater_than("1.5.0"): + defer_env_var = "DBT_DEFER" + state_env_var = "DBT_STATE" + else: + defer_env_var = "DBT_DEFER_TO_STATE" + state_env_var = "DBT_ARTIFACT_STATE_PATH" + if defer: - env_vars['DBT_DEFER_TO_STATE'] = 'true' - env_vars['DBT_ARTIFACT_STATE_PATH'] = plugin_config.dbt_artifacts_prod + env_vars[defer_env_var] = 'true' + env_vars[state_env_var] = plugin_config.dbt_artifacts_prod else: - env_vars['DBT_ARTIFACT_STATE_PATH'] = plugin_config.dbt_artifacts_local + env_vars[state_env_var] = plugin_config.dbt_artifacts_local return env_vars From 3d2369de2ac1f8b9d889d21d8c18ec87e7f0207a Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Fri, 8 Sep 2023 15:31:38 -0400 Subject: [PATCH 06/11] Check version equality by default, but make it optional --- palm/plugins/dbt/plugin_config.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/palm/plugins/dbt/plugin_config.py b/palm/plugins/dbt/plugin_config.py index aa90591..47dab9c 100644 --- a/palm/plugins/dbt/plugin_config.py +++ b/palm/plugins/dbt/plugin_config.py @@ -16,12 +16,16 @@ class dbtPluginConfigModel(BaseModel): def dbt_version_semver(self) -> str: return semver.Version.parse(self.dbt_version) - def is_dbt_version_greater_than(self, version: str) -> bool: + def is_dbt_version_greater_than(self, version: str, or_equal: bool = True) -> bool: target_version = semver.Version.parse(version) + if or_equal: + return self.dbt_version_semver() >= target_version return self.dbt_version_semver() > target_version - def is_dbt_version_less_than(self, version: str) -> bool: + def is_dbt_version_less_than(self, version: str, or_equal: bool = True) -> bool: target_version = semver.Version.parse(version) + if or_equal: + return self.dbt_version_semver() <= target_version return self.dbt_version_semver() < target_version From 902face1fea7f3cf26295e945447fea14ef0eb41 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Fri, 8 Sep 2023 15:33:10 -0400 Subject: [PATCH 07/11] Bump minimum palm version dependency Changes depend on a refactor to run_in_docker that will be part of the next minor palm release. --- palm/plugins/dbt/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/palm/plugins/dbt/requirements.txt b/palm/plugins/dbt/requirements.txt index d37e902..db1ae85 100644 --- a/palm/plugins/dbt/requirements.txt +++ b/palm/plugins/dbt/requirements.txt @@ -1,4 +1,4 @@ -palm >=2.5.1, <3.0 +palm >=2.6.0, <3.0 pyyaml >= 5.0 sqlparse >= 0.3.1 semver >= 3.0.1 From c99e335349e59bf7305a8909cffef6d262c62205 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Wed, 13 Sep 2023 17:05:05 -0400 Subject: [PATCH 08/11] Refactor: Use default factory to get the version and update palm config for existing users. --- palm/plugins/dbt/dbt_version_detection.py | 54 +++++++++++++++++++---- palm/plugins/dbt/plugin_config.py | 11 +++-- 2 files changed, 53 insertions(+), 12 deletions(-) diff --git a/palm/plugins/dbt/dbt_version_detection.py b/palm/plugins/dbt/dbt_version_detection.py index f581a71..f97f1b3 100644 --- a/palm/plugins/dbt/dbt_version_detection.py +++ b/palm/plugins/dbt/dbt_version_detection.py @@ -1,25 +1,63 @@ -from email.mime import image +import click from pathlib import Path import yaml from palm.utils import run_in_docker -def get_dbt_version(): +def get_dbt_version() -> str: + """Get the dbt version from the dbt-core package in the docker image. + + Returns: + str: The dbt version + """ image_name = _get_image_name() cmd = "pip list | grep dbt-core" - success, msg = run_in_docker(cmd, image_name, capture_output=True) + success, msg = run_in_docker(cmd, image_name, capture_output=True, silent=True) if not success: raise ValueError(f"Error getting dbt version: {msg}") return msg.strip().split(" ")[-1] +def dbt_version_factory() -> str: + """Get the dbt version from the dbt-core package in the docker image. + Then update the dbt version in the .palm/config.yaml file. + + Returns: + str: The dbt version + """ + click.secho("Detecting dbt version...", fg="yellow") + version = get_dbt_version() + set_dbt_version_in_config(version) + click.secho( + f"Detected dbt version: {version}. .palm/config.yml updated" + , fg="green" + ) + return version + +def set_dbt_version_in_config(version: str) -> None: + """Update the dbt version in the .palm/config.yaml file.""" + palm_config = _get_palm_config() + palm_config['plugin_config']['dbt']['dbt_version'] = version + _update_palm_config(palm_config) def _get_image_name(): - """Get the image name from the .palm/config.yaml file. + """Get the image name from the .palm/config.yaml file.""" + palm_config = _get_palm_config() + return palm_config["image_name"] - This isn't great, but we don't want to instantiate a PalmConfig object here. - """ +# Ideally, the 2 functions below would come from palm_config.py in palm-cli +# However, that class is more complicated than it should be and needs a refactor. +# So, for now, we have to do this here. + +def _get_palm_config() -> dict: + """Get the palm config from the .palm/config.yaml file.""" palm_config_path = Path(".palm/config.yaml") if not palm_config_path.exists(): raise ValueError("No .palm/config.yaml file found") - palm_config = yaml.safe_load(palm_config_path.read_text()) - return palm_config["image_name"] + return yaml.safe_load(palm_config_path.read_text()) + +def _update_palm_config(config: dict) -> None: + """Update the palm config in the .palm/config.yaml file.""" + palm_config_path = Path(".palm/config.yaml") + if not palm_config_path.exists(): + raise ValueError("No .palm/config.yaml file found") + palm_config_path.write_text(yaml.dump(config)) diff --git a/palm/plugins/dbt/plugin_config.py b/palm/plugins/dbt/plugin_config.py index 47dab9c..58082af 100644 --- a/palm/plugins/dbt/plugin_config.py +++ b/palm/plugins/dbt/plugin_config.py @@ -2,16 +2,17 @@ from typing import Optional from pathlib import Path +import yaml import semver -from pydantic import BaseModel +from pydantic import BaseModel, Field from palm.plugins.base_plugin_config import BasePluginConfig -from palm.plugins.dbt.dbt_version_detection import get_dbt_version +from palm.plugins.dbt.dbt_version_detection import dbt_version_factory, get_dbt_version class dbtPluginConfigModel(BaseModel): dbt_artifacts_prod: Optional[str] dbt_artifacts_local: str - dbt_version: str + dbt_version: str = Field(default_factory=dbt_version_factory) def dbt_version_semver(self) -> str: return semver.Version.parse(self.dbt_version) @@ -41,7 +42,9 @@ def set(self) -> dict: ) if has_prod_artifacts: prod = click.prompt( - "Prod artifacts location:", type=click.Path(exists=True) + "Prod artifacts location:" + , type=click.Path(exists=True) + , default=Path("target/prod_artifacts") ) config["dbt_artifacts_prod"] = str(prod) click.secho(f"Saved prod artifacts location: {prod}", fg="green") From 9b10fc1cfa7b11306c6641a0b4d09890b10834a6 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Wed, 11 Oct 2023 09:52:04 -0400 Subject: [PATCH 09/11] Linter fixes --- palm/plugins/dbt/dbt_version_detection.py | 10 ++++++++-- palm/plugins/dbt/plugin_config.py | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/palm/plugins/dbt/dbt_version_detection.py b/palm/plugins/dbt/dbt_version_detection.py index f97f1b3..ba59ad9 100644 --- a/palm/plugins/dbt/dbt_version_detection.py +++ b/palm/plugins/dbt/dbt_version_detection.py @@ -4,6 +4,7 @@ from palm.utils import run_in_docker + def get_dbt_version() -> str: """Get the dbt version from the dbt-core package in the docker image. @@ -17,6 +18,7 @@ def get_dbt_version() -> str: raise ValueError(f"Error getting dbt version: {msg}") return msg.strip().split(" ")[-1] + def dbt_version_factory() -> str: """Get the dbt version from the dbt-core package in the docker image. Then update the dbt version in the .palm/config.yaml file. @@ -28,26 +30,29 @@ def dbt_version_factory() -> str: version = get_dbt_version() set_dbt_version_in_config(version) click.secho( - f"Detected dbt version: {version}. .palm/config.yml updated" - , fg="green" + f"Detected dbt version: {version}. .palm/config.yml updated", fg="green" ) return version + def set_dbt_version_in_config(version: str) -> None: """Update the dbt version in the .palm/config.yaml file.""" palm_config = _get_palm_config() palm_config['plugin_config']['dbt']['dbt_version'] = version _update_palm_config(palm_config) + def _get_image_name(): """Get the image name from the .palm/config.yaml file.""" palm_config = _get_palm_config() return palm_config["image_name"] + # Ideally, the 2 functions below would come from palm_config.py in palm-cli # However, that class is more complicated than it should be and needs a refactor. # So, for now, we have to do this here. + def _get_palm_config() -> dict: """Get the palm config from the .palm/config.yaml file.""" palm_config_path = Path(".palm/config.yaml") @@ -55,6 +60,7 @@ def _get_palm_config() -> dict: raise ValueError("No .palm/config.yaml file found") return yaml.safe_load(palm_config_path.read_text()) + def _update_palm_config(config: dict) -> None: """Update the palm config in the .palm/config.yaml file.""" palm_config_path = Path(".palm/config.yaml") diff --git a/palm/plugins/dbt/plugin_config.py b/palm/plugins/dbt/plugin_config.py index 58082af..a680e50 100644 --- a/palm/plugins/dbt/plugin_config.py +++ b/palm/plugins/dbt/plugin_config.py @@ -42,9 +42,9 @@ def set(self) -> dict: ) if has_prod_artifacts: prod = click.prompt( - "Prod artifacts location:" - , type=click.Path(exists=True) - , default=Path("target/prod_artifacts") + "Prod artifacts location:", + type=click.Path(exists=True), + default=Path("target/prod_artifacts"), ) config["dbt_artifacts_prod"] = str(prod) click.secho(f"Saved prod artifacts location: {prod}", fg="green") From 122fbb28cf41a9b4f06eebe0bcf94c6829c2fde3 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Mon, 6 Nov 2023 09:37:42 -0500 Subject: [PATCH 10/11] Add or_equal to version checks for 1.5.0 --- palm/plugins/dbt/commands/cmd_run.py | 12 ++++++++++-- palm/plugins/dbt/commands/cmd_test.py | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/palm/plugins/dbt/commands/cmd_run.py b/palm/plugins/dbt/commands/cmd_run.py index 5d3bae2..17cde8b 100644 --- a/palm/plugins/dbt/commands/cmd_run.py +++ b/palm/plugins/dbt/commands/cmd_run.py @@ -24,6 +24,7 @@ help="Will perform a full refresh on incremental models", ) @click.option("--seed", is_flag=True, help="Run dbt seed before dbt run") +@click.option("--lightdash", '-ld', is_flag=True, help="Run dbt with lightdash enabled") @click.pass_obj def cli( environment, @@ -33,6 +34,7 @@ def cli( iterative: bool, defer: bool, seed: bool, + lightdash: bool, models: Optional[Tuple] = tuple(), select: Optional[Tuple] = tuple(), exclude: Optional[Tuple] = tuple(), @@ -70,6 +72,7 @@ def cli( exclude=exclude, defer=defer, vars=vars, + lightdash=lightdash, ) success, msg = environment.run_in_docker(run_cmd, env_vars) @@ -107,6 +110,7 @@ def build_run_command( exclude: Optional[Tuple] = None, defer: bool = False, vars: Optional[str] = None, + lightdash: bool = False, ) -> str: cmd = [] full_refresh_option = " --full-refresh" if full_refresh else "" @@ -115,7 +119,11 @@ def build_run_command( cmd.append(f"dbt seed --full-refresh") cmd.append("&&") - cmd.append(f"dbt run{full_refresh_option}") + if lightdash: + cmd.append(f"lightdash dbt run{full_refresh_option}") + else: + cmd.append(f"dbt run{full_refresh_option}") + if targets: cmd.append("--select") cmd.extend(targets) @@ -139,7 +147,7 @@ def set_env_vars(environment, stateful: bool, defer: bool = False) -> dict: env_vars = dbt_env_vars(environment.palm.branch) # These env vars are renamed in dbt v1.5.0, old env vars are deprecated - if plugin_config.is_dbt_version_greater_than("1.5.0"): + if plugin_config.is_dbt_version_greater_than("1.5.0", or_equal=True): defer_env_var = "DBT_DEFER" state_env_var = "DBT_STATE" else: diff --git a/palm/plugins/dbt/commands/cmd_test.py b/palm/plugins/dbt/commands/cmd_test.py index d0f7414..d6609cf 100644 --- a/palm/plugins/dbt/commands/cmd_test.py +++ b/palm/plugins/dbt/commands/cmd_test.py @@ -93,7 +93,7 @@ def set_env_vars(environment, defer: bool = False) -> dict: env_vars = dbt_env_vars(environment.palm.branch) # These env vars are renamed in dbt v1.5.0, old env vars are deprecated - if plugin_config.is_dbt_version_greater_than("1.5.0"): + if plugin_config.is_dbt_version_greater_than("1.5.0", or_equal=True): defer_env_var = "DBT_DEFER" state_env_var = "DBT_STATE" else: From 3ffe9827449c7c8658aec90ada8c8197017f87b1 Mon Sep 17 00:00:00 2001 From: Jake Beresford Date: Mon, 20 Nov 2023 10:33:54 -0500 Subject: [PATCH 11/11] Prepare for release: bump version and update changleog --- CHANGELOG.md | 15 +++++++++++++++ setup.py | 2 +- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd41208..6fa7896 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.8.0] - 2023-11-20 + +### Added + +- dbt version detection: palm can now detect the version of dbt used in a project + and will add the version to the plugin config. This can then be used to support + changes between dbt versions. +- support for dbt's yaml selectors via the `--selector` option on relevant commands. + +### Changed + +- Requires palm v2.6.0 or higher +- env vars are updated for projects running dbt v1.5.0 and later. This resolves + deprecation warnings for people using more recent versions of dbt. + ## [0.7.0] - 2023-06-13 ### Added diff --git a/setup.py b/setup.py index ab36ef5..34bce03 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ setup( name='palm-dbt', - version='0.7.0', + version='0.8.0', description='dbt extension for Palm CLI', long_description=long_description, long_description_content_type='text/markdown',