From 42c3572de7dd497b83708d61e2aa4d731714b566 Mon Sep 17 00:00:00 2001 From: Nat Noordanus Date: Sat, 22 Jun 2024 16:57:11 +0200 Subject: [PATCH] Add suppoort for POE_GIT_DIR and POE_GIT_ROOT variables in config --- docs/env_vars.rst | 10 +++ docs/global_options.rst | 17 ++++- docs/guides/include_guide.rst | 13 ++++ docs/tasks/options.rst | 8 +++ poethepoet/config.py | 35 +++++++++- poethepoet/env/cache.py | 2 +- poethepoet/env/manager.py | 44 +++++++++--- poethepoet/helpers/git.py | 66 ++++++++++++++++++ poethepoet/task/cmd.py | 4 +- tests/fixtures/includes_project/git_repo | 1 + .../includes_project/only_includes.toml | 2 +- .../fixtures/includes_project/pyproject.toml | 1 - tests/test_includes.py | 68 +++++++++++++++++++ 13 files changed, 252 insertions(+), 19 deletions(-) create mode 100644 poethepoet/helpers/git.py create mode 160000 tests/fixtures/includes_project/git_repo diff --git a/docs/env_vars.rst b/docs/env_vars.rst index e3140bae6..63fede66e 100644 --- a/docs/env_vars.rst +++ b/docs/env_vars.rst @@ -11,6 +11,16 @@ The following environment variables are used by Poe the Poet internally, and can - ``POE_CONF_DIR``: the path to the parent directory of the config file that defines the running task or the :ref:`cwd option` set when including that config. - ``POE_ACTIVE``: identifies the active PoeExecutor, so that Poe the Poet can tell when it is running recursively. + +Special variables +----------------- + +The following variables are not set on the environment by default but can be referenced from task configuration as if they were. + +- ``POE_GIT_DIR``: path of the git repo that the project is part of. This allows a project in a subdirectory of a monorepo to reference :ref:`includes` or :ref:`envfiles` relative to the root of the git repo. Note that referencing this variable causes poe to attempt to call the ``git`` executable which must be available on the path. + +- ``POE_GIT_ROOT``: just like ``POE_GIT_DIR`` except that if the project is in a git submodule, then the path will point to the working directory of the main repo above it. + External Environment variables ------------------------------ diff --git a/docs/global_options.rst b/docs/global_options.rst index e3bf6f01a..500c2106c 100644 --- a/docs/global_options.rst +++ b/docs/global_options.rst @@ -74,7 +74,22 @@ You can also specify an env file (with bash-like syntax) to load for all tasks l [tool.poe] envfile = ".env" -The envfile global option also accepts a list of env files. +The envfile global option also accepts a list of env files like so. + +.. code-block:: toml + [tool.poe] + envfile = ["standard.env", "local.env"] + +In this case the referenced files will be loaded in the given order. + +Normally envfile paths are resolved relative to the project root (that is the parent directory of the pyproject.toml). However when working with a monorepo it can also be useful to specify the path relative to the root of the git repository, which can be done by referenceing the ``POE_GIT_DIR`` or ``POE_GIT_ROOT`` variables like so: + +.. code-block:: toml + + [tool.poe] + envfile = "${POE_GIT_DIR}/.env" + +See the documentation on :ref:`Special variables` for a full explanation of how these variables work. Change the executor type ------------------------ diff --git a/docs/guides/include_guide.rst b/docs/guides/include_guide.rst index bb15314b2..0f980c76d 100644 --- a/docs/guides/include_guide.rst +++ b/docs/guides/include_guide.rst @@ -63,3 +63,16 @@ You can still specify that an envfile referenced within an included file should [tool.poe] envfile = "${POE_ROOT}/.env" + + +Including files relative to the git repo +---------------------------------------- + +Normally include paths are resolved relative to the project root (that is the parent directory of the pyproject.toml). However when working with a monorepo it can also be useful to specify the file to include relative to the root of the git repository, which can be done by referenceing the ``POE_GIT_DIR`` or ``POE_GIT_ROOT`` variables like so: + +.. code-block:: toml + + [tool.poe] + include = "${POE_GIT_DIR}/tasks.toml" + +See the documentation on :ref:`Special variables` for a full explanation of how these variables work. diff --git a/docs/tasks/options.rst b/docs/tasks/options.rst index a32897fac..efd9fe713 100644 --- a/docs/tasks/options.rst +++ b/docs/tasks/options.rst @@ -103,6 +103,14 @@ above but can also by given a list of such paths like so: In this case the referenced files will be loaded in the given order. +Normally envfile paths are resolved relative to the project root (that is the parent directory of the pyproject.toml). However when working with a monorepo it can also be useful to specify the path relative to the root of the git repository, which can be done by referenceing the ``POE_GIT_DIR`` or ``POE_GIT_ROOT`` variables like so: + +.. code-block:: toml + + [tool.poe] + envfile = "${POE_GIT_DIR}/.env" + +See the documentation on :ref:`Special variables` for a full explanation of how these variables work. Running a task with a specific working directory ------------------------------------------------ diff --git a/poethepoet/config.py b/poethepoet/config.py index 6826a9358..f865a66f4 100644 --- a/poethepoet/config.py +++ b/poethepoet/config.py @@ -1,4 +1,5 @@ import json +from os import environ from pathlib import Path from types import MappingProxyType @@ -23,6 +24,8 @@ from .exceptions import ConfigValidationError, PoeException from .options import NoValue, PoeOptions +POE_DEBUG = environ.get("POE_DEBUG", "0") == "1" + class ConfigPartition: options: PoeOptions @@ -427,7 +430,7 @@ def find_config_file( if not target_path.exists(): raise PoeException( f"Poe could not find a {self._config_name!r} file at the given " - f"location: {target_path!r}" + f"location: {str(target_path)!r}" ) return target_path @@ -455,11 +458,14 @@ def _resolve_project_dir(self, target_dir: Path, raise_on_fail: bool = False): def _load_includes(self: "PoeConfig", strict: bool = True): # Attempt to load each of the included configs for include in self._project_config.options.include: - include_path = self._project_dir.joinpath(include["path"]).resolve() + include_path = self._resolve_include_path(include["path"]) if not include_path.exists(): # TODO: print warning in verbose mode, requires access to ui somehow # Maybe there should be something like a WarningService? + + if POE_DEBUG: + print(f" ! Could not include file from invalid path {include_path}") continue try: @@ -476,12 +482,37 @@ def _load_includes(self: "PoeConfig", strict: bool = True): strict=strict, ) ) + if POE_DEBUG: + print(f" Included config from {include_path}") except (PoeException, KeyError) as error: raise ConfigValidationError( f"Invalid content in included file from {include_path}", filename=str(include_path), ) from error + def _resolve_include_path(self, include_path: str): + from .env.template import apply_envvars_to_template + + available_vars = {"POE_ROOT": str(self._project_dir)} + + if "${POE_GIT_DIR}" in include_path: + from .helpers.git import GitRepo + + git_repo = GitRepo(self._project_dir) + available_vars["POE_GIT_DIR"] = str(git_repo.path or "") + + if "${POE_GIT_ROOT}" in include_path: + from .helpers.git import GitRepo + + git_repo = GitRepo(self._project_dir) + available_vars["POE_GIT_ROOT"] = str(git_repo.main_path or "") + + include_path = apply_envvars_to_template( + include_path, available_vars, require_braces=True + ) + + return self._project_dir.joinpath(include_path).resolve() + @staticmethod def _read_config_file(path: Path) -> Mapping[str, Any]: try: diff --git a/poethepoet/env/cache.py b/poethepoet/env/cache.py index 9b85139a0..5c930da30 100644 --- a/poethepoet/env/cache.py +++ b/poethepoet/env/cache.py @@ -35,7 +35,7 @@ def get(self, envfile: Union[str, Path]) -> Dict[str, str]: with envfile_path.open(encoding="utf-8") as envfile_file: result = parse_env_file(envfile_file.readlines()) if POE_DEBUG: - print(f" - Loaded Envfile from {envfile_path}") + print(f" + Loaded Envfile from {envfile_path}") except ValueError as error: message = error.args[0] raise ExecutionError( diff --git a/poethepoet/env/manager.py b/poethepoet/env/manager.py index f692876ee..e9c478afd 100644 --- a/poethepoet/env/manager.py +++ b/poethepoet/env/manager.py @@ -10,7 +10,7 @@ from .ui import PoeUi -class EnvVarsManager: +class EnvVarsManager(Mapping): _config: "PoeConfig" _ui: Optional["PoeUi"] _vars: Dict[str, str] @@ -24,13 +24,14 @@ def __init__( # TODO: check if we still need all these args! base_env: Optional[Mapping[str, str]] = None, cwd: Optional[Union[Path, str]] = None, ): + from ..helpers.git import GitRepo from .cache import EnvFileCache self._config = config self._ui = ui self.envfiles = ( # Reuse EnvFileCache from parent_env when possible - EnvFileCache(Path(config.project_dir), self._ui) + EnvFileCache(config.project_dir, self._ui) if parent_env is None else parent_env.envfiles ) @@ -39,14 +40,33 @@ def __init__( # TODO: check if we still need all these args! **(base_env or {}), } - self._vars["POE_ROOT"] = str(self._config.project_dir) + self._vars["POE_ROOT"] = str(config.project_dir) self.cwd = str(cwd or os.getcwd()) if "POE_CWD" not in self._vars: self._vars["POE_CWD"] = self.cwd - self._vars["POE_PWD"] = self.cwd # Deprecated + self._vars["POE_PWD"] = self.cwd + + self._git_repo = GitRepo(config.project_dir) + + def __getitem__(self, key): + return self._vars[key] + + def __iter__(self): + return iter(self._vars) + + def __len__(self): + return len(self._vars) + + def get(self, key: Any, /, default: Any = None) -> Optional[str]: + if key == "POE_GIT_DIR": + # This is a special case environment variable that is only set if requested + self._vars["POE_GIT_DIR"] = str(self._git_repo.path or "") + + if key == "POE_GIT_ROOT": + # This is a special case environment variable that is only set if requested + self._vars["POE_GIT_ROOT"] = str(self._git_repo.main_path or "") - def get(self, key: str, default: Optional[str] = None) -> Optional[str]: return self._vars.get(key, default) def set(self, key: str, value: str): @@ -65,7 +85,9 @@ def apply_env_config( used if the associated key doesn't already have a value. """ - vars_scope = dict(self._vars, POE_CONF_DIR=str(config_dir)) + scoped_env = self.clone() + scoped_env.set("POE_CONF_DIR", str(config_dir)) + if envfile: if isinstance(envfile, str): envfile = [envfile] @@ -74,23 +96,25 @@ def apply_env_config( self.envfiles.get( config_working_dir.joinpath( apply_envvars_to_template( - envfile_path, vars_scope, require_braces=True + envfile_path, scoped_env, require_braces=True ) ) ) ) - vars_scope = dict(self._vars, POE_CONF_DIR=str(config_dir)) + scoped_env = self.clone() + scoped_env.set("POE_CONF_DIR", str(config_dir)) + for key, value in (config_env or {}).items(): if isinstance(value, str): value_str = value - elif key not in vars_scope: + elif key not in scoped_env: value_str = value["default"] else: continue self._vars[key] = apply_envvars_to_template( - value_str, vars_scope, require_braces=True + value_str, scoped_env, require_braces=True ) def update(self, env_vars: Mapping[str, Any]): diff --git a/poethepoet/helpers/git.py b/poethepoet/helpers/git.py new file mode 100644 index 000000000..f76767742 --- /dev/null +++ b/poethepoet/helpers/git.py @@ -0,0 +1,66 @@ +import shutil +from pathlib import Path +from subprocess import PIPE, Popen +from typing import Optional, Tuple + + +class GitRepo: + def __init__(self, seed_path: Path): + self._seed_path = seed_path + self._path: Optional[Path] = None + self._main_path: Optional[Path] = None + + @property + def path(self) -> Optional[Path]: + if self._path is None: + self._path = self._resolve_path() + return self._path + + @property + def main_path(self) -> Optional[Path]: + if self._main_path is None: + self._main_path = self._resolve_main_path() + return self._main_path + + def init(self): + self._exec("init") + + def delete_git_dir(self): + shutil.rmtree(self._seed_path.joinpath(".git")) + + def _resolve_path(self) -> Optional[Path]: + """ + Resolve the path of this git repo + """ + proc, captured_stdout = self._exec( + "rev-parse", "--show-superproject-working-tree", "--show-toplevel" + ) + if proc.returncode == 0: + captured_lines = ( + line.strip() for line in captured_stdout.decode().strip().split("\n") + ) + longest_line = sorted((len(line), line) for line in captured_lines)[-1][1] + return Path(longest_line) + return None + + def _resolve_main_path(self) -> Optional[Path]: + """ + Resolve the path of this git repo, unless this repo is a git submodule, + then resolve the path of the main git repo. + """ + proc, captured_stdout = self._exec( + "rev-parse", "--show-superproject-working-tree", "--show-toplevel" + ) + if proc.returncode == 0: + return Path(captured_stdout.decode().strip().split("\n")[0]) + return None + + def _exec(self, *args: str) -> Tuple[Popen, bytes]: + proc = Popen( + ["git", *args], + cwd=self._seed_path, + stdout=PIPE, + stderr=PIPE, + ) + (captured_stdout, _) = proc.communicate() + return proc, captured_stdout diff --git a/poethepoet/task/cmd.py b/poethepoet/task/cmd.py index fd175623e..52fa2b451 100644 --- a/poethepoet/task/cmd.py +++ b/poethepoet/task/cmd.py @@ -84,9 +84,7 @@ def _resolve_commandline(self, context: "RunContext", env: "EnvVarsManager"): working_dir = self.get_working_dir(env) result = [] - for cmd_token, has_glob in resolve_command_tokens( - command_lines[0], env.to_dict() - ): + for cmd_token, has_glob in resolve_command_tokens(command_lines[0], env): if has_glob: # Resolve glob pattern from the working directory result.extend([str(match) for match in working_dir.glob(cmd_token)]) diff --git a/tests/fixtures/includes_project/git_repo b/tests/fixtures/includes_project/git_repo new file mode 160000 index 000000000..3049d3871 --- /dev/null +++ b/tests/fixtures/includes_project/git_repo @@ -0,0 +1 @@ +Subproject commit 3049d38712a19460d110c1b60303cd1931401480 diff --git a/tests/fixtures/includes_project/only_includes.toml b/tests/fixtures/includes_project/only_includes.toml index fe34fc55f..eef4e644c 100644 --- a/tests/fixtures/includes_project/only_includes.toml +++ b/tests/fixtures/includes_project/only_includes.toml @@ -1,2 +1,2 @@ [tool.poe] -include = "greet.toml" +include = "${POE_GIT_ROOT}/tests/fixtures/includes_project/greet.toml" diff --git a/tests/fixtures/includes_project/pyproject.toml b/tests/fixtures/includes_project/pyproject.toml index 00113924f..df824cc02 100644 --- a/tests/fixtures/includes_project/pyproject.toml +++ b/tests/fixtures/includes_project/pyproject.toml @@ -1,7 +1,6 @@ [tool.poe] include = "greet.toml" - [tool.poe.tasks.echo] cmd = "poe_test_echo" help = "says what you say" diff --git a/tests/test_includes.py b/tests/test_includes.py index 3e4ba44ae..e7f4de084 100644 --- a/tests/test_includes.py +++ b/tests/test_includes.py @@ -1,3 +1,19 @@ +import pytest + + +@pytest.fixture(scope="session") +def _init_git_repo(projects): + from poethepoet.helpers.git import GitRepo + + repo_path = projects["includes/git_repo/sub_project"].parent.parent + repo = GitRepo(repo_path) + # Init the git repo for use in tests + repo.init() + yield + # Delete the .git dir so that we don't have to deal with it as a git submodule + repo.delete_git_dir() + + def test_docs_for_include_toml_file(run_poe_subproc): result = run_poe_subproc(project="includes") assert ( @@ -306,3 +322,55 @@ def test_include_subproject_envfiles_with_cwd_set( "TASK_REL_SOURCE_CONFIG: task level rel to source config\n" ) assert result.stderr == "" + + +@pytest.mark.usefixtures("_init_git_repo") +def test_include_tasks_from_git_repo(run_poe_subproc, projects): + # test task included relative to POE_GIT_ROOT + result = run_poe_subproc( + "did_it_work", cwd=projects["includes/git_repo/sub_project"] + ) + assert result.capture == "Poe => poe_test_echo yes\n" + assert result.stdout == "yes\n" + assert result.stderr == "" + + # test task included relative to POE_GIT_DIR + result = run_poe_subproc( + "did_it_work2", cwd=projects["includes/git_repo/sub_project"] + ) + assert result.capture == "Poe => poe_test_echo yes\n" + assert result.stdout == "yes\n" + assert result.stderr == "" + + +@pytest.mark.usefixtures("_init_git_repo") +def test_use_poe_git_vars(run_poe_subproc, projects, is_windows, poe_project_path): + result = run_poe_subproc( + "has_repo_env_vars", cwd=projects["includes/git_repo/sub_project"] + ) + assert result.capture.startswith("Poe => poe_test_echo XXX") + if is_windows: + assert result.stdout.endswith( + f"XXX {poe_project_path} " + f"YYY {poe_project_path}\\tests\\fixtures\\includes_project\\git_repo ZZZ\n" + ) + else: + assert result.stdout.endswith( + f"XXX {poe_project_path} " + f"YYY {poe_project_path}/tests/fixtures/includes_project/git_repo ZZZ\n" + ) + assert result.stderr == "" + + +@pytest.mark.usefixtures("_init_git_repo") +def test_poe_git_vars_for_task_level_envfile_and_env( + run_poe_subproc, projects, poe_project_path +): + result = run_poe_subproc("print_env", cwd=projects["includes/git_repo/sub_project"]) + assert result.capture.startswith("Poe => poe_test_env") + assert "POE_GIT_ROOT=" not in result.stdout + assert f"POE_GIT_ROOT_2={poe_project_path}" in result.stdout + assert "POE_GIT_DIR=" not in result.stdout + assert f"POE_GIT_DIR_2={poe_project_path}" in result.stdout + assert "BASE_ENV_LOADED=" in result.stdout + assert result.stderr == ""