From daf8d5a1bf1c369b6624c5279f70bf842c2b8efb Mon Sep 17 00:00:00 2001 From: Pawel Job Date: Thu, 30 Jan 2025 17:02:30 +0100 Subject: [PATCH 1/2] Build PyApp binary --- build_binary.py | 158 ++++++++++++++++++++++++++++++++++++++++++ build_pyapp_binary.sh | 4 ++ pyproject.toml | 3 + 3 files changed, 165 insertions(+) create mode 100644 build_binary.py create mode 100755 build_pyapp_binary.sh diff --git a/build_binary.py b/build_binary.py new file mode 100644 index 0000000000..4abcfa5ad6 --- /dev/null +++ b/build_binary.py @@ -0,0 +1,158 @@ +""" +Use hatch to build a binary that doesn't require any network connection. + +Installs a Python distribution to a dir in TMP; makes and installs the project +wheel into that distribution; makes a bzipped archive of the distribution; then +builds the binary with the distribution embedded. + +Run this script from the project root dir. +""" + +import contextlib +import json +import os +import subprocess +import tarfile +import tempfile +from pathlib import Path + +import tomllib + + +class ProjectSettings: + """Class for holding dynamically determined build settings.""" + + def __init__(self) -> None: + data: dict = self.get_pyproject_data() + self.pyproject_data = data + self.project_name: str = data["project"]["name"] + self.python_version: str = data["tool"]["hatch"]["build"]["targets"]["binary"][ + "python-version" + ] + self.project_version: str = self.get_project_version() + self.python_tmp_dir = Path( + tempfile.gettempdir(), f"{self.project_name}-{self.project_version}" + ) + self.python_dist_root_version = Path(self.python_tmp_dir / self.python_version) + # This is the path to the python executable within the distribution archive + self.__python_path_within_archive: Path | None = None + + @staticmethod + def get_project_version() -> str: + """Use hatch to get the project version.""" + completed_proc = subprocess.run(["hatch", "version"], capture_output=True) + return completed_proc.stdout.decode().strip() + + @staticmethod + def get_pyproject_data() -> dict: + """Retrieve the pyproject.toml data.""" + with open("pyproject.toml", mode="rb") as fp: + data = tomllib.load(fp) + return data + + @property + def python_path_within_archive(self) -> Path: + """Returns the path to the root of the Python dist that we'll bundle.""" + if self.__python_path_within_archive is None: + with (self.python_dist_root_version / "hatch-dist.json").open() as fp: + hatch_json = json.load(fp) + self.__python_path_within_archive = hatch_json["python_path"] + return self.__python_path_within_archive + + @property + def python_dist_exe(self) -> Path: + """The full path to the distribution's python executable. + + Used for running 'pip install'. + """ + return self.python_dist_root_version / self.python_path_within_archive + + +def make_project_wheel() -> Path: + """Return path to the project's wheel build.""" + completed_proc = subprocess.run( + ["hatch", "build", "-t", "wheel"], capture_output=True + ) + return Path(completed_proc.stderr.decode().strip()) + + +def make_dist_archive(python_tmp_dir: Path, dist_path: Path) -> Path: + """Make and return path to tar-bzipped Python distribution.""" + archive = python_tmp_dir / "python.bz2" + with contextlib.chdir(dist_path): + with tarfile.open(archive, mode="w:bz2") as tar: + tar.add(".") + return archive + + +def hatch_install_python(python_tmp_dir: Path, python_version: str) -> bool: + """Install Python dist into temp dir for bundling.""" + completed_proc = subprocess.run( + [ + "hatch", + "python", + "install", + "--private", + "--dir", + python_tmp_dir, + python_version, + ] + ) + return not completed_proc.returncode + + +def pip_install_project(python_exe: str, project_whl: Path) -> bool: + """Install the project into the Python distribution.""" + completed_proc = subprocess.run( + [python_exe, "-m", "pip", "install", "-U", str(project_whl)], + capture_output=True, + ) + return not completed_proc.returncode + + +def hatch_build_binary(archive_path: Path, python_path: Path) -> Path | None: + """Use hatch to build the binary.""" + os.environ["PYAPP_SKIP_INSTALL"] = "1" + os.environ["PYAPP_DISTRIBUTION_PATH"] = str(archive_path) + os.environ["PYAPP_FULL_ISOLATION"] = "1" + os.environ["PYAPP_DISTRIBUTION_PYTHON_PATH"] = str(python_path) + completed_proc = subprocess.run( + ["hatch", "build", "-t", "binary"], capture_output=True + ) + if completed_proc.returncode: + print(completed_proc.stderr) + return None + # The binary location is the last line of stderr + return Path(completed_proc.stderr.decode().split()[-1]) + + +def main(): + settings = ProjectSettings() + print("Installing Python distribution to TMP dir...") + hatch_install_python(settings.python_tmp_dir, settings.python_version) + print("-> installed") + + print("Building wheel...") + project_wheel = make_project_wheel() + print("->", project_wheel) + + print(f"Installing {project_wheel} into Python distribution...") + pip_install_project(str(settings.python_dist_exe), project_wheel) + print("-> installed") + + print("Making distribution archive...") + archive_path = make_dist_archive( + settings.python_tmp_dir, settings.python_dist_root_version + ) + print("->", archive_path) + + print(f"Building '{settings.project_name}' binary...") + binary_location = hatch_build_binary( + archive_path, settings.python_path_within_archive + ) + if binary_location: + print("-> binary location:", binary_location) + + +if __name__ == "__main__": + main() diff --git a/build_pyapp_binary.sh b/build_pyapp_binary.sh new file mode 100755 index 0000000000..6baff5639a --- /dev/null +++ b/build_pyapp_binary.sh @@ -0,0 +1,4 @@ +pip install -U hatch +hatch build +pip install git+https://github.com/hobbsd/hatch-build-isolated-binary.git@v1.0.0 +build-binary diff --git a/pyproject.toml b/pyproject.toml index ec4992a863..152c4bae79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -188,3 +188,6 @@ markers = [ [tool.codespell] skip = 'tests/*,snow.spec' + +[tool.hatch.build.targets.binary] +python-version = "3.11" From c77db98bdf467e543178eee4f25789d08080995a Mon Sep 17 00:00:00 2001 From: Pawel Job Date: Thu, 30 Jan 2025 18:34:18 +0100 Subject: [PATCH 2/2] Plugins in external directory - working poc --- build_binary.py | 158 ------------------ build_pyapp_binary.sh | 1 + src/snowflake/cli/__about__.py | 2 +- .../_app/commands_registration/__init__.py | 3 + src/snowflake/cli/_plugins/plugin/commands.py | 29 +++- 5 files changed, 33 insertions(+), 160 deletions(-) delete mode 100644 build_binary.py diff --git a/build_binary.py b/build_binary.py deleted file mode 100644 index 4abcfa5ad6..0000000000 --- a/build_binary.py +++ /dev/null @@ -1,158 +0,0 @@ -""" -Use hatch to build a binary that doesn't require any network connection. - -Installs a Python distribution to a dir in TMP; makes and installs the project -wheel into that distribution; makes a bzipped archive of the distribution; then -builds the binary with the distribution embedded. - -Run this script from the project root dir. -""" - -import contextlib -import json -import os -import subprocess -import tarfile -import tempfile -from pathlib import Path - -import tomllib - - -class ProjectSettings: - """Class for holding dynamically determined build settings.""" - - def __init__(self) -> None: - data: dict = self.get_pyproject_data() - self.pyproject_data = data - self.project_name: str = data["project"]["name"] - self.python_version: str = data["tool"]["hatch"]["build"]["targets"]["binary"][ - "python-version" - ] - self.project_version: str = self.get_project_version() - self.python_tmp_dir = Path( - tempfile.gettempdir(), f"{self.project_name}-{self.project_version}" - ) - self.python_dist_root_version = Path(self.python_tmp_dir / self.python_version) - # This is the path to the python executable within the distribution archive - self.__python_path_within_archive: Path | None = None - - @staticmethod - def get_project_version() -> str: - """Use hatch to get the project version.""" - completed_proc = subprocess.run(["hatch", "version"], capture_output=True) - return completed_proc.stdout.decode().strip() - - @staticmethod - def get_pyproject_data() -> dict: - """Retrieve the pyproject.toml data.""" - with open("pyproject.toml", mode="rb") as fp: - data = tomllib.load(fp) - return data - - @property - def python_path_within_archive(self) -> Path: - """Returns the path to the root of the Python dist that we'll bundle.""" - if self.__python_path_within_archive is None: - with (self.python_dist_root_version / "hatch-dist.json").open() as fp: - hatch_json = json.load(fp) - self.__python_path_within_archive = hatch_json["python_path"] - return self.__python_path_within_archive - - @property - def python_dist_exe(self) -> Path: - """The full path to the distribution's python executable. - - Used for running 'pip install'. - """ - return self.python_dist_root_version / self.python_path_within_archive - - -def make_project_wheel() -> Path: - """Return path to the project's wheel build.""" - completed_proc = subprocess.run( - ["hatch", "build", "-t", "wheel"], capture_output=True - ) - return Path(completed_proc.stderr.decode().strip()) - - -def make_dist_archive(python_tmp_dir: Path, dist_path: Path) -> Path: - """Make and return path to tar-bzipped Python distribution.""" - archive = python_tmp_dir / "python.bz2" - with contextlib.chdir(dist_path): - with tarfile.open(archive, mode="w:bz2") as tar: - tar.add(".") - return archive - - -def hatch_install_python(python_tmp_dir: Path, python_version: str) -> bool: - """Install Python dist into temp dir for bundling.""" - completed_proc = subprocess.run( - [ - "hatch", - "python", - "install", - "--private", - "--dir", - python_tmp_dir, - python_version, - ] - ) - return not completed_proc.returncode - - -def pip_install_project(python_exe: str, project_whl: Path) -> bool: - """Install the project into the Python distribution.""" - completed_proc = subprocess.run( - [python_exe, "-m", "pip", "install", "-U", str(project_whl)], - capture_output=True, - ) - return not completed_proc.returncode - - -def hatch_build_binary(archive_path: Path, python_path: Path) -> Path | None: - """Use hatch to build the binary.""" - os.environ["PYAPP_SKIP_INSTALL"] = "1" - os.environ["PYAPP_DISTRIBUTION_PATH"] = str(archive_path) - os.environ["PYAPP_FULL_ISOLATION"] = "1" - os.environ["PYAPP_DISTRIBUTION_PYTHON_PATH"] = str(python_path) - completed_proc = subprocess.run( - ["hatch", "build", "-t", "binary"], capture_output=True - ) - if completed_proc.returncode: - print(completed_proc.stderr) - return None - # The binary location is the last line of stderr - return Path(completed_proc.stderr.decode().split()[-1]) - - -def main(): - settings = ProjectSettings() - print("Installing Python distribution to TMP dir...") - hatch_install_python(settings.python_tmp_dir, settings.python_version) - print("-> installed") - - print("Building wheel...") - project_wheel = make_project_wheel() - print("->", project_wheel) - - print(f"Installing {project_wheel} into Python distribution...") - pip_install_project(str(settings.python_dist_exe), project_wheel) - print("-> installed") - - print("Making distribution archive...") - archive_path = make_dist_archive( - settings.python_tmp_dir, settings.python_dist_root_version - ) - print("->", archive_path) - - print(f"Building '{settings.project_name}' binary...") - binary_location = hatch_build_binary( - archive_path, settings.python_path_within_archive - ) - if binary_location: - print("-> binary location:", binary_location) - - -if __name__ == "__main__": - main() diff --git a/build_pyapp_binary.sh b/build_pyapp_binary.sh index 6baff5639a..2851dc4ad6 100755 --- a/build_pyapp_binary.sh +++ b/build_pyapp_binary.sh @@ -1,4 +1,5 @@ pip install -U hatch +hatch clean hatch build pip install git+https://github.com/hobbsd/hatch-build-isolated-binary.git@v1.0.0 build-binary diff --git a/src/snowflake/cli/__about__.py b/src/snowflake/cli/__about__.py index 0847d0bcd8..4017a4d7ab 100644 --- a/src/snowflake/cli/__about__.py +++ b/src/snowflake/cli/__about__.py @@ -14,4 +14,4 @@ from __future__ import annotations -VERSION = "3.4.0.dev0" +VERSION = "3.4.0.dev8" diff --git a/src/snowflake/cli/_app/commands_registration/__init__.py b/src/snowflake/cli/_app/commands_registration/__init__.py index c28b33e082..a2dd10255f 100644 --- a/src/snowflake/cli/_app/commands_registration/__init__.py +++ b/src/snowflake/cli/_app/commands_registration/__init__.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys from dataclasses import dataclass from snowflake.cli.api.plugins.command import CommandSpec +sys.path.append("/tmp/cli_plugins/lib/python3.11/site-packages") + @dataclass class LoadedCommandPlugin: diff --git a/src/snowflake/cli/_plugins/plugin/commands.py b/src/snowflake/cli/_plugins/plugin/commands.py index 8b8c8f3d99..46628df95c 100644 --- a/src/snowflake/cli/_plugins/plugin/commands.py +++ b/src/snowflake/cli/_plugins/plugin/commands.py @@ -15,6 +15,8 @@ from __future__ import annotations import logging +import subprocess +import sys import typer from snowflake.cli.api.commands.snow_typer import SnowTyperFactory @@ -26,10 +28,35 @@ app = SnowTyperFactory( name="plugin", help="Plugin management commands.", - is_hidden=lambda: True, ) +@app.command(name="install", requires_connection=False) +def install( + package: str = typer.Argument( + None, help="Pip compatible package (PyPI name / URL / local path)" + ), + **options, +) -> CommandResult: + """Installs a plugin from a package""" + target_dir: str = ( + "/tmp/cli_plugins" # TODO make it configurable in config.toml with some default + ) + subprocess.check_call( + [ + sys.executable, + "-m", + "pip", + "install", + "--prefix", + target_dir, + "--upgrade", + package, + ] + ) + return MessageResult(f"Plugin successfully installed.") + + @app.command(name="enable", requires_connection=False) def enable( plugin_name: str = typer.Argument(None, help="Plugin name"),