From 86e6e8a2fac7e1fa73790e81490cefdde0ea4086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 8 Sep 2024 20:22:09 +0200 Subject: [PATCH 01/28] Install conda_subprocess in test environment --- .github/workflows/unittests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 5d28fc3..d56edd0 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -49,4 +49,7 @@ jobs: pip install versioneer[toml]==0.29 pip install . --no-deps --no-build-isolation conda create -y -n py312 python=3.12.1 + conda activate py312 + pip install . --no-deps --no-build-isolation + conda deactivate python -m unittest discover tests From b17228b936e176bd35392cba78ae0a92fc2015eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 8 Sep 2024 20:24:22 +0200 Subject: [PATCH 02/28] Install versioneer --- .github/workflows/unittests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index d56edd0..fc2ffc0 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -50,6 +50,7 @@ jobs: pip install . --no-deps --no-build-isolation conda create -y -n py312 python=3.12.1 conda activate py312 + pip install versioneer[toml]==0.29 pip install . --no-deps --no-build-isolation conda deactivate python -m unittest discover tests From 23298b28d4c59463648cb615204c93b3f1248a35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 8 Sep 2024 20:28:01 +0200 Subject: [PATCH 03/28] Install conda --- .github/workflows/unittests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index fc2ffc0..589d52d 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -48,7 +48,7 @@ jobs: run: | pip install versioneer[toml]==0.29 pip install . --no-deps --no-build-isolation - conda create -y -n py312 python=3.12.1 + conda create -y -n py312 python=3.12.1 conda=24.7.1 conda activate py312 pip install versioneer[toml]==0.29 pip install . --no-deps --no-build-isolation From d1903879f9756f3f006da75437c41c4bf1f6a008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 8 Sep 2024 21:49:03 +0200 Subject: [PATCH 04/28] Add version test --- .ci_support/environment.yml | 1 + .github/workflows/unittests.yml | 2 +- tests/test_conda_function.py | 36 +++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 tests/test_conda_function.py diff --git a/.ci_support/environment.yml b/.ci_support/environment.yml index a7cf1bd..cd9a191 100644 --- a/.ci_support/environment.yml +++ b/.ci_support/environment.yml @@ -3,3 +3,4 @@ channels: dependencies: - python - conda =24.7.1 +- executorlib =0.0.2 \ No newline at end of file diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index 589d52d..8707c23 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -48,7 +48,7 @@ jobs: run: | pip install versioneer[toml]==0.29 pip install . --no-deps --no-build-isolation - conda create -y -n py312 python=3.12.1 conda=24.7.1 + conda create -y -n py312 python=3.12.1 conda=24.7.1 executorlib=0.0.2 conda activate py312 pip install versioneer[toml]==0.29 pip install . --no-deps --no-build-isolation diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py new file mode 100644 index 0000000..bdbabe2 --- /dev/null +++ b/tests/test_conda_function.py @@ -0,0 +1,36 @@ +from concurrent.futures import Future +import queue +from unittest import TestCase +from executorlib.shared.executor import execute_parallel_tasks +from executorlib.shared.interface import SubprocessInterface +from executorlib.shared.executor import cloudpickle_register + + +def add_function(parameter_1, parameter_2): + import sys + return parameter_1 + parameter_2 + sys.version_info.major + sys.version_info.minor + + +class TestCondaFunction(TestCase): + def test_conda_function(self): + cloudpickle_register(ind=1) + task_queue = queue.Queue() + task_future = Future() + task_queue.put( + { + "fn": add_function, + "args": (), + "kwargs": {"parameter_1": 1, "parameter_2": 2}, + "future": task_future, + "resource_dict": {"cores": 1}, + } + ) + task_queue.put({"shutdown": True, "wait": True}) + execute_parallel_tasks( + future_queue=task_queue, + interface_class=SubprocessInterface(), + max_cores=1, + hostname_localhost=False, + prefix_name="py312", + ) + self.assertEqual(task_future.result(), 18) From 68c87a575bbb85fcf2c00b934ec179b74a6d84d4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 8 Sep 2024 19:50:09 +0000 Subject: [PATCH 05/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_conda_function.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index bdbabe2..f7af89e 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -8,6 +8,7 @@ def add_function(parameter_1, parameter_2): import sys + return parameter_1 + parameter_2 + sys.version_info.major + sys.version_info.minor From 230d9150e001218beb8bb52c7de6dc22a69bc69e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 8 Sep 2024 21:50:50 +0200 Subject: [PATCH 06/28] fixes --- tests/test_conda_function.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index bdbabe2..62606f0 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -28,7 +28,7 @@ def test_conda_function(self): task_queue.put({"shutdown": True, "wait": True}) execute_parallel_tasks( future_queue=task_queue, - interface_class=SubprocessInterface(), + interface_class=SubprocessInterface, max_cores=1, hostname_localhost=False, prefix_name="py312", From 6504e5befc3c419132e8545a42984448c9264fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 8 Sep 2024 21:52:33 +0200 Subject: [PATCH 07/28] max_cores to cors --- tests/test_conda_function.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index dea9774..0a27dc6 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -30,7 +30,7 @@ def test_conda_function(self): execute_parallel_tasks( future_queue=task_queue, interface_class=SubprocessInterface, - max_cores=1, + cores=1, hostname_localhost=False, prefix_name="py312", ) From 7b639a619b68b981600e55b2b5d6ba8670fcfbfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 8 Sep 2024 22:56:17 +0200 Subject: [PATCH 08/28] fix backend --- tests/test_conda_function.py | 62 +++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index 0a27dc6..4824d57 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -1,9 +1,9 @@ from concurrent.futures import Future import queue from unittest import TestCase -from executorlib.shared.executor import execute_parallel_tasks from executorlib.shared.interface import SubprocessInterface from executorlib.shared.executor import cloudpickle_register +from executorlib.shared.communication import interface_bootup def add_function(parameter_1, parameter_2): @@ -12,6 +12,66 @@ def add_function(parameter_1, parameter_2): return parameter_1 + parameter_2 + sys.version_info.major + sys.version_info.minor +def execute_parallel_tasks( + future_queue, + cores, + interface_class, + hostname_localhost, + init_function, + prefix_name, + prefix_path, + **kwargs, +) -> None: + """ + Execute a single tasks in parallel using the message passing interface (MPI). + + Args: + future_queue (queue.Queue): task queue of dictionary objects which are submitted to the parallel process + cores (int): defines the total number of MPI ranks to use + interface_class (BaseInterface): Interface to start process on selected compute resources + hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the + context of an HPC cluster this essential to be able to communicate to an + Executor running on a different compute node within the same allocation. And + in principle any computer should be able to resolve that their own hostname + points to the same address as localhost. Still MacOS >= 12 seems to disable + this look up for security reasons. So on MacOS it is required to set this + option to true + init_function (callable): optional function to preset arguments for functions which are submitted later + prefix_name (str): name of the conda environment to initialize + prefix_path (str): path of the conda environment to initialize + """ + interface = interface_bootup( + command_lst=["python"], + connections=interface_class(cores=cores, **kwargs), + hostname_localhost=hostname_localhost, + prefix_path=prefix_path, + prefix_name=prefix_name, + ) + if init_function is not None: + interface.send_dict( + input_dict={"init": True, "fn": init_function, "args": (), "kwargs": {}} + ) + while True: + task_dict = future_queue.get() + if "shutdown" in task_dict.keys() and task_dict["shutdown"]: + interface.shutdown(wait=task_dict["wait"]) + future_queue.task_done() + future_queue.join() + break + elif "fn" in task_dict.keys() and "future" in task_dict.keys(): + f = task_dict.pop("future") + if f.set_running_or_notify_cancel(): + try: + f.set_result(interface.send_and_receive_dict(input_dict=task_dict)) + except Exception as thread_exception: + interface.shutdown(wait=True) + future_queue.task_done() + f.set_exception(exception=thread_exception) + raise thread_exception + else: + future_queue.task_done() + + class TestCondaFunction(TestCase): def test_conda_function(self): cloudpickle_register(ind=1) From f938ee6942a31b5cfa88e0fef55dc5910b2d0e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 8 Sep 2024 22:58:43 +0200 Subject: [PATCH 09/28] defaults --- tests/test_conda_function.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index 4824d57..ea7ee38 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -16,10 +16,10 @@ def execute_parallel_tasks( future_queue, cores, interface_class, - hostname_localhost, - init_function, - prefix_name, - prefix_path, + hostname_localhost=False, + init_function=None, + prefix_name=None, + prefix_path=None, **kwargs, ) -> None: """ From 08d345f59e907308c13186638be3badf9ddec477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 8 Sep 2024 23:02:23 +0200 Subject: [PATCH 10/28] fix command path --- tests/test_conda_function.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index ea7ee38..e275965 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -2,7 +2,7 @@ import queue from unittest import TestCase from executorlib.shared.interface import SubprocessInterface -from executorlib.shared.executor import cloudpickle_register +from executorlib.shared.executor import cloudpickle_register, get_command_path from executorlib.shared.communication import interface_bootup @@ -41,7 +41,7 @@ def execute_parallel_tasks( prefix_path (str): path of the conda environment to initialize """ interface = interface_bootup( - command_lst=["python"], + command_lst=["python", get_command_path(executable="interactive_serial.py")], connections=interface_class(cores=cores, **kwargs), hostname_localhost=hostname_localhost, prefix_path=prefix_path, From 40d1bb671201a04f21e567d402a52b24cf327d3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 8 Sep 2024 23:06:02 +0200 Subject: [PATCH 11/28] importlib --- tests/test_conda_function.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index e275965..af9c732 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -7,9 +7,10 @@ def add_function(parameter_1, parameter_2): - import sys + import importlib + system = importlib.import_module('sys') - return parameter_1 + parameter_2 + sys.version_info.major + sys.version_info.minor + return parameter_1 + parameter_2 + system.version_info.major + system.version_info.minor def execute_parallel_tasks( @@ -17,7 +18,6 @@ def execute_parallel_tasks( cores, interface_class, hostname_localhost=False, - init_function=None, prefix_name=None, prefix_path=None, **kwargs, @@ -36,7 +36,6 @@ def execute_parallel_tasks( points to the same address as localhost. Still MacOS >= 12 seems to disable this look up for security reasons. So on MacOS it is required to set this option to true - init_function (callable): optional function to preset arguments for functions which are submitted later prefix_name (str): name of the conda environment to initialize prefix_path (str): path of the conda environment to initialize """ @@ -47,10 +46,6 @@ def execute_parallel_tasks( prefix_path=prefix_path, prefix_name=prefix_name, ) - if init_function is not None: - interface.send_dict( - input_dict={"init": True, "fn": init_function, "args": (), "kwargs": {}} - ) while True: task_dict = future_queue.get() if "shutdown" in task_dict.keys() and task_dict["shutdown"]: From b588c425fc735db89640104b0cce2410deb84005 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 8 Sep 2024 21:06:10 +0000 Subject: [PATCH 12/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_conda_function.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index af9c732..fa3c1c0 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -8,9 +8,15 @@ def add_function(parameter_1, parameter_2): import importlib - system = importlib.import_module('sys') - return parameter_1 + parameter_2 + system.version_info.major + system.version_info.minor + system = importlib.import_module("sys") + + return ( + parameter_1 + + parameter_2 + + system.version_info.major + + system.version_info.minor + ) def execute_parallel_tasks( From 879b651a1d6f200c5a8146f380d4ca5757b4733f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 8 Sep 2024 23:12:28 +0200 Subject: [PATCH 13/28] call interface directly --- tests/test_conda_function.py | 79 ++++++------------------------------ 1 file changed, 12 insertions(+), 67 deletions(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index fa3c1c0..ede56f4 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -19,80 +19,25 @@ def add_function(parameter_1, parameter_2): ) -def execute_parallel_tasks( - future_queue, - cores, - interface_class, - hostname_localhost=False, - prefix_name=None, - prefix_path=None, - **kwargs, -) -> None: - """ - Execute a single tasks in parallel using the message passing interface (MPI). - - Args: - future_queue (queue.Queue): task queue of dictionary objects which are submitted to the parallel process - cores (int): defines the total number of MPI ranks to use - interface_class (BaseInterface): Interface to start process on selected compute resources - hostname_localhost (boolean): use localhost instead of the hostname to establish the zmq connection. In the - context of an HPC cluster this essential to be able to communicate to an - Executor running on a different compute node within the same allocation. And - in principle any computer should be able to resolve that their own hostname - points to the same address as localhost. Still MacOS >= 12 seems to disable - this look up for security reasons. So on MacOS it is required to set this - option to true - prefix_name (str): name of the conda environment to initialize - prefix_path (str): path of the conda environment to initialize - """ - interface = interface_bootup( - command_lst=["python", get_command_path(executable="interactive_serial.py")], - connections=interface_class(cores=cores, **kwargs), - hostname_localhost=hostname_localhost, - prefix_path=prefix_path, - prefix_name=prefix_name, - ) - while True: - task_dict = future_queue.get() - if "shutdown" in task_dict.keys() and task_dict["shutdown"]: - interface.shutdown(wait=task_dict["wait"]) - future_queue.task_done() - future_queue.join() - break - elif "fn" in task_dict.keys() and "future" in task_dict.keys(): - f = task_dict.pop("future") - if f.set_running_or_notify_cancel(): - try: - f.set_result(interface.send_and_receive_dict(input_dict=task_dict)) - except Exception as thread_exception: - interface.shutdown(wait=True) - future_queue.task_done() - f.set_exception(exception=thread_exception) - raise thread_exception - else: - future_queue.task_done() - - class TestCondaFunction(TestCase): def test_conda_function(self): cloudpickle_register(ind=1) task_queue = queue.Queue() task_future = Future() - task_queue.put( - { - "fn": add_function, - "args": (), - "kwargs": {"parameter_1": 1, "parameter_2": 2}, - "future": task_future, - "resource_dict": {"cores": 1}, - } - ) + task_dict = { + "fn": add_function, + "args": (), + "kwargs": {"parameter_1": 1, "parameter_2": 2}, + "resource_dict": {"cores": 1}, + } task_queue.put({"shutdown": True, "wait": True}) - execute_parallel_tasks( - future_queue=task_queue, - interface_class=SubprocessInterface, - cores=1, + interface = interface_bootup( + command_lst=["python", get_command_path(executable="interactive_serial.py")], + connections=SubprocessInterface(cores=1), hostname_localhost=False, + prefix_path=None, prefix_name="py312", ) + task_future.set_result(interface.send_and_receive_dict(input_dict=task_dict)) + interface.shutdown(wait=True) self.assertEqual(task_future.result(), 18) From d960d276050b04d3514e6d853a4e9449e212a843 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 8 Sep 2024 21:12:56 +0000 Subject: [PATCH 14/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_conda_function.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index ede56f4..f6d1920 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -32,7 +32,10 @@ def test_conda_function(self): } task_queue.put({"shutdown": True, "wait": True}) interface = interface_bootup( - command_lst=["python", get_command_path(executable="interactive_serial.py")], + command_lst=[ + "python", + get_command_path(executable="interactive_serial.py"), + ], connections=SubprocessInterface(cores=1), hostname_localhost=False, prefix_path=None, From a979d414b161fc6e9449c59c642176469f844425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 8 Sep 2024 23:17:16 +0200 Subject: [PATCH 15/28] maybe sys is a bad module --- tests/test_conda_function.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index ede56f4..008461e 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -7,15 +7,11 @@ def add_function(parameter_1, parameter_2): - import importlib - - system = importlib.import_module("sys") + import os return ( - parameter_1 - + parameter_2 - + system.version_info.major - + system.version_info.minor + parameter_1 + parameter_2, + os.environ["CONDA_PREFIX"] ) @@ -40,4 +36,6 @@ def test_conda_function(self): ) task_future.set_result(interface.send_and_receive_dict(input_dict=task_dict)) interface.shutdown(wait=True) - self.assertEqual(task_future.result(), 18) + number, prefix = task_future.result() + print(prefix) + self.assertEqual(number, 3) From c5f1298ba493f10ef4bc23ebad37660774fc834a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 8 Sep 2024 21:17:30 +0000 Subject: [PATCH 16/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_conda_function.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index 4997757..b9cd856 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -9,10 +9,7 @@ def add_function(parameter_1, parameter_2): import os - return ( - parameter_1 + parameter_2, - os.environ["CONDA_PREFIX"] - ) + return (parameter_1 + parameter_2, os.environ["CONDA_PREFIX"]) class TestCondaFunction(TestCase): From cd79cfc82fdb9749f6201783d7453edc8d6d2fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 8 Sep 2024 23:23:46 +0200 Subject: [PATCH 17/28] skip test if not Python 3.12 --- tests/test_conda_function.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index 4997757..c46452d 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -1,6 +1,7 @@ from concurrent.futures import Future import queue -from unittest import TestCase +import sys +from unittest import TestCase, skipIf from executorlib.shared.interface import SubprocessInterface from executorlib.shared.executor import cloudpickle_register, get_command_path from executorlib.shared.communication import interface_bootup @@ -15,6 +16,7 @@ def add_function(parameter_1, parameter_2): ) +skipIf(sys.version_info.minor != 12, "Test environment has to be Python 3.12 for consistency.") class TestCondaFunction(TestCase): def test_conda_function(self): cloudpickle_register(ind=1) @@ -40,5 +42,5 @@ def test_conda_function(self): task_future.set_result(interface.send_and_receive_dict(input_dict=task_dict)) interface.shutdown(wait=True) number, prefix = task_future.result() - print(prefix) + self.assertEqual(prefix[-5:], "py312") self.assertEqual(number, 3) From b349519cdff1a632a8c353074e1108ec435f680a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 8 Sep 2024 21:24:00 +0000 Subject: [PATCH 18/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_conda_function.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index 3482d6d..33eca6c 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -13,7 +13,12 @@ def add_function(parameter_1, parameter_2): return (parameter_1 + parameter_2, os.environ["CONDA_PREFIX"]) -skipIf(sys.version_info.minor != 12, "Test environment has to be Python 3.12 for consistency.") +skipIf( + sys.version_info.minor != 12, + "Test environment has to be Python 3.12 for consistency.", +) + + class TestCondaFunction(TestCase): def test_conda_function(self): cloudpickle_register(ind=1) From 2208a26e1ccb81c41b23012e43e9286ba7b15f6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 8 Sep 2024 23:33:20 +0200 Subject: [PATCH 19/28] use conda_subprocess --- tests/test_conda_function.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index 3482d6d..7dce1ab 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -1,10 +1,15 @@ from concurrent.futures import Future import queue +from socket import gethostname +import subprocess import sys -from unittest import TestCase, skipIf +import unittest + from executorlib.shared.interface import SubprocessInterface from executorlib.shared.executor import cloudpickle_register, get_command_path -from executorlib.shared.communication import interface_bootup +from executorlib.shared.communication import SocketInterface + +import conda_subprocess def add_function(parameter_1, parameter_2): @@ -13,8 +18,8 @@ def add_function(parameter_1, parameter_2): return (parameter_1 + parameter_2, os.environ["CONDA_PREFIX"]) -skipIf(sys.version_info.minor != 12, "Test environment has to be Python 3.12 for consistency.") -class TestCondaFunction(TestCase): +@unittest.skipIf(sys.version_info.minor != 12, "Test environment has to be Python 3.12 for consistency.") +class TestCondaFunction(unittest.TestCase): def test_conda_function(self): cloudpickle_register(ind=1) task_queue = queue.Queue() @@ -26,14 +31,19 @@ def test_conda_function(self): "resource_dict": {"cores": 1}, } task_queue.put({"shutdown": True, "wait": True}) - interface = interface_bootup( - command_lst=[ - "python", - get_command_path(executable="interactive_serial.py"), - ], - connections=SubprocessInterface(cores=1), - hostname_localhost=False, - prefix_path=None, + interface = SocketInterface(interface=SubprocessInterface(cores=1)) + command_lst = [ + "python", + get_command_path(executable="interactive_serial.py"), + "--host", + gethostname(), + "--zmqport", + str(interface.bind_to_random_port()), + ] + interface._interface._process = conda_subprocess.Popen( + args=interface._interface.generate_command(command_lst=command_lst), + cwd=interface._interface._cwd, + stdin=subprocess.DEVNULL, prefix_name="py312", ) task_future.set_result(interface.send_and_receive_dict(input_dict=task_dict)) From 1dcba6c68c82a1bbe0304b816dd7eacbffbb0355 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 8 Sep 2024 21:33:48 +0000 Subject: [PATCH 20/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_conda_function.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index 7dce1ab..b57fadd 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -18,7 +18,10 @@ def add_function(parameter_1, parameter_2): return (parameter_1 + parameter_2, os.environ["CONDA_PREFIX"]) -@unittest.skipIf(sys.version_info.minor != 12, "Test environment has to be Python 3.12 for consistency.") +@unittest.skipIf( + sys.version_info.minor != 12, + "Test environment has to be Python 3.12 for consistency.", +) class TestCondaFunction(unittest.TestCase): def test_conda_function(self): cloudpickle_register(ind=1) From 949ce11da468276d816dda6b8f2635b64d64e452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Sun, 8 Sep 2024 23:38:41 +0200 Subject: [PATCH 21/28] do not require executorlib --- tests/test_conda_function.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index b57fadd..72b887f 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -5,9 +5,12 @@ import sys import unittest -from executorlib.shared.interface import SubprocessInterface -from executorlib.shared.executor import cloudpickle_register, get_command_path -from executorlib.shared.communication import SocketInterface +try: + from executorlib.shared.interface import SubprocessInterface + from executorlib.shared.executor import cloudpickle_register, get_command_path + from executorlib.shared.communication import SocketInterface +except ImportError: + pass import conda_subprocess From 6bbbb842da1e3fc7219f93a2fdedf0eb77beef59 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Mon, 9 Sep 2024 09:27:56 +0200 Subject: [PATCH 22/28] switch to decorator --- conda_subprocess/decorator.py | 47 +++++++++++++++++++++++++++++++++++ tests/test_conda_function.py | 40 ++++------------------------- 2 files changed, 52 insertions(+), 35 deletions(-) create mode 100644 conda_subprocess/decorator.py diff --git a/conda_subprocess/decorator.py b/conda_subprocess/decorator.py new file mode 100644 index 0000000..3fd2def --- /dev/null +++ b/conda_subprocess/decorator.py @@ -0,0 +1,47 @@ +from concurrent.futures import Future +from socket import gethostname +import subprocess +from typing import Optional + +try: + from executorlib.shared.interface import SubprocessInterface + from executorlib.shared.executor import get_command_path + from executorlib.shared.communication import SocketInterface +except ImportError: + pass + +from conda_subprocess.process import Popen + + +def conda(prefix_name: Optional[str]=None, prefix_path: Optional[str]=None): + def conda_function(funct): + def function_wrapped(*args, **kwargs): + task_future = Future() + task_dict = { + "fn": funct, + "args": args, + "kwargs": kwargs, + "resource_dict": {"cores": 1}, + } + interface = SocketInterface(interface=SubprocessInterface(cores=1)) + command_lst = [ + "python", + get_command_path(executable="interactive_serial.py"), + "--host", + gethostname(), + "--zmqport", + str(interface.bind_to_random_port()), + ] + interface._interface._process = Popen( + args=interface._interface.generate_command(command_lst=command_lst), + cwd=interface._interface._cwd, + stdin=subprocess.DEVNULL, + prefix_name=prefix_name, + prefix_path=prefix_path, + ) + task_future.set_result(interface.send_and_receive_dict(input_dict=task_dict)) + interface.shutdown(wait=True) + return task_future.result() + + return function_wrapped + return conda_function diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index 72b887f..9e27535 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -1,20 +1,16 @@ -from concurrent.futures import Future -import queue -from socket import gethostname -import subprocess import sys import unittest try: - from executorlib.shared.interface import SubprocessInterface - from executorlib.shared.executor import cloudpickle_register, get_command_path - from executorlib.shared.communication import SocketInterface + from executorlib.shared.executor import cloudpickle_register except ImportError: pass -import conda_subprocess +from conda_subprocess.decorator import conda + +@conda(prefix_name="py312") def add_function(parameter_1, parameter_2): import os @@ -28,32 +24,6 @@ def add_function(parameter_1, parameter_2): class TestCondaFunction(unittest.TestCase): def test_conda_function(self): cloudpickle_register(ind=1) - task_queue = queue.Queue() - task_future = Future() - task_dict = { - "fn": add_function, - "args": (), - "kwargs": {"parameter_1": 1, "parameter_2": 2}, - "resource_dict": {"cores": 1}, - } - task_queue.put({"shutdown": True, "wait": True}) - interface = SocketInterface(interface=SubprocessInterface(cores=1)) - command_lst = [ - "python", - get_command_path(executable="interactive_serial.py"), - "--host", - gethostname(), - "--zmqport", - str(interface.bind_to_random_port()), - ] - interface._interface._process = conda_subprocess.Popen( - args=interface._interface.generate_command(command_lst=command_lst), - cwd=interface._interface._cwd, - stdin=subprocess.DEVNULL, - prefix_name="py312", - ) - task_future.set_result(interface.send_and_receive_dict(input_dict=task_dict)) - interface.shutdown(wait=True) - number, prefix = task_future.result() + number, prefix = add_function(parameter_1=1, parameter_2=2) self.assertEqual(prefix[-5:], "py312") self.assertEqual(number, 3) From d5a3b5ff066a1e0b504402d0c6dc15a9785af0b3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 9 Sep 2024 07:28:04 +0000 Subject: [PATCH 23/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- conda_subprocess/decorator.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/conda_subprocess/decorator.py b/conda_subprocess/decorator.py index 3fd2def..d749bc4 100644 --- a/conda_subprocess/decorator.py +++ b/conda_subprocess/decorator.py @@ -1,19 +1,19 @@ +import subprocess from concurrent.futures import Future from socket import gethostname -import subprocess from typing import Optional try: - from executorlib.shared.interface import SubprocessInterface - from executorlib.shared.executor import get_command_path from executorlib.shared.communication import SocketInterface + from executorlib.shared.executor import get_command_path + from executorlib.shared.interface import SubprocessInterface except ImportError: pass from conda_subprocess.process import Popen -def conda(prefix_name: Optional[str]=None, prefix_path: Optional[str]=None): +def conda(prefix_name: Optional[str] = None, prefix_path: Optional[str] = None): def conda_function(funct): def function_wrapped(*args, **kwargs): task_future = Future() @@ -39,9 +39,12 @@ def function_wrapped(*args, **kwargs): prefix_name=prefix_name, prefix_path=prefix_path, ) - task_future.set_result(interface.send_and_receive_dict(input_dict=task_dict)) + task_future.set_result( + interface.send_and_receive_dict(input_dict=task_dict) + ) interface.shutdown(wait=True) return task_future.result() return function_wrapped + return conda_function From 12911d6787e57fe3733d768ab91f6e3524403369 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 10 Sep 2024 08:31:53 +0200 Subject: [PATCH 24/28] Fix import --- conda_subprocess/decorator.py | 9 +++------ pyproject.toml | 5 +++++ tests/test_conda_function.py | 11 +++++++---- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/conda_subprocess/decorator.py b/conda_subprocess/decorator.py index d749bc4..f4a84c1 100644 --- a/conda_subprocess/decorator.py +++ b/conda_subprocess/decorator.py @@ -3,12 +3,9 @@ from socket import gethostname from typing import Optional -try: - from executorlib.shared.communication import SocketInterface - from executorlib.shared.executor import get_command_path - from executorlib.shared.interface import SubprocessInterface -except ImportError: - pass +from executorlib.shared.communication import SocketInterface +from executorlib.shared.executor import get_command_path +from executorlib.shared.interface import SubprocessInterface from conda_subprocess.process import Popen diff --git a/pyproject.toml b/pyproject.toml index 49f2fe5..5fb73d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,11 @@ dynamic = ["version"] [project.urls] Repository = "https://github.com/pyiron/conda_subprocess" +[project.optional-dependencies] +executorlib = [ + "executorlib==0.0.2", +] + [tool.setuptools.packages.find] include = ["conda_subprocess*"] diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index 9e27535..30aab54 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -2,12 +2,15 @@ import unittest try: + from conda_subprocess.decorator import conda from executorlib.shared.executor import cloudpickle_register except ImportError: - pass - - -from conda_subprocess.decorator import conda + def conda(prefix_name=None, prefix_path=None): + def wrap_function(funct): + def function_out(*args, **kwargs): + return None + return function_out + return wrap_function @conda(prefix_name="py312") From a01e19b2b91ecfee459129cdc5bbb34d66dd0f06 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 10 Sep 2024 06:32:48 +0000 Subject: [PATCH 25/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_conda_function.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_conda_function.py b/tests/test_conda_function.py index 30aab54..ed75f19 100644 --- a/tests/test_conda_function.py +++ b/tests/test_conda_function.py @@ -5,11 +5,14 @@ from conda_subprocess.decorator import conda from executorlib.shared.executor import cloudpickle_register except ImportError: + def conda(prefix_name=None, prefix_path=None): def wrap_function(funct): def function_out(*args, **kwargs): return None + return function_out + return wrap_function From 578726d9bea382cf0a90d0c7cc2dbecd82f804f5 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 10 Sep 2024 08:47:22 +0200 Subject: [PATCH 26/28] Add hostname_localhost parameter --- conda_subprocess/decorator.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/conda_subprocess/decorator.py b/conda_subprocess/decorator.py index f4a84c1..7feea78 100644 --- a/conda_subprocess/decorator.py +++ b/conda_subprocess/decorator.py @@ -10,7 +10,7 @@ from conda_subprocess.process import Popen -def conda(prefix_name: Optional[str] = None, prefix_path: Optional[str] = None): +def conda(prefix_name: Optional[str] = None, prefix_path: Optional[str] = None, hostname_localhost: bool = False): def conda_function(funct): def function_wrapped(*args, **kwargs): task_future = Future() @@ -21,14 +21,10 @@ def function_wrapped(*args, **kwargs): "resource_dict": {"cores": 1}, } interface = SocketInterface(interface=SubprocessInterface(cores=1)) - command_lst = [ - "python", - get_command_path(executable="interactive_serial.py"), - "--host", - gethostname(), - "--zmqport", - str(interface.bind_to_random_port()), - ] + command_lst = ["python", get_command_path(executable="interactive_serial.py")] + if not hostname_localhost: + command_lst += ["--host", gethostname()] + command_lst += ["--zmqport", str(interface.bind_to_random_port())] interface._interface._process = Popen( args=interface._interface.generate_command(command_lst=command_lst), cwd=interface._interface._cwd, From 1e35c568a7a4287de5771b2b47db3af76ed702f4 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 10 Sep 2024 08:49:53 +0200 Subject: [PATCH 27/28] black fixes --- conda_subprocess/decorator.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/conda_subprocess/decorator.py b/conda_subprocess/decorator.py index 7feea78..1c65802 100644 --- a/conda_subprocess/decorator.py +++ b/conda_subprocess/decorator.py @@ -10,7 +10,11 @@ from conda_subprocess.process import Popen -def conda(prefix_name: Optional[str] = None, prefix_path: Optional[str] = None, hostname_localhost: bool = False): +def conda( + prefix_name: Optional[str] = None, + prefix_path: Optional[str] = None, + hostname_localhost: bool = False, +): def conda_function(funct): def function_wrapped(*args, **kwargs): task_future = Future() @@ -21,7 +25,10 @@ def function_wrapped(*args, **kwargs): "resource_dict": {"cores": 1}, } interface = SocketInterface(interface=SubprocessInterface(cores=1)) - command_lst = ["python", get_command_path(executable="interactive_serial.py")] + command_lst = [ + "python", + get_command_path(executable="interactive_serial.py"), + ] if not hostname_localhost: command_lst += ["--host", gethostname()] command_lst += ["--zmqport", str(interface.bind_to_random_port())] From f801eaaee7b8f7e16d809b3c11716b5609934fc6 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Tue, 10 Sep 2024 09:26:09 +0200 Subject: [PATCH 28/28] Add README.md --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index cc42f19..d04ea6f 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Create a new conda environment - in this example a conda environment for Python conda create -n py312 python=3.12 ``` +### Subprocess Interface Open a python shell in your base environment where `conda_subprocess` is installed and execute `python --version` in the `py312` environment: ```python @@ -64,6 +65,20 @@ process.communicate() >>> (b'Python 3.12.1\n', None) ``` +### Decorator +In analogy to the subprocess interface the `conda_subprocess` also introduces the `@conda` decorator to +execute python functions in a separate conda environment: +```python +from conda_subprocess.decorator import conda + +@conda(prefix_name="py312") +def add_function(parameter_1, parameter_2): + return parameter_1 + parameter_2 + +add_function(parameter_1=1, parameter_2=2) +>>> 3 +``` + ## Remarks * The `shell` parameter and the `env` parameter are not supported in `Popen()` and all derived methods. * The `pipesize` parameter and the `process_group` parameter were removed for compatibility with python 3.9.