From 9704b7c2f5ddeaa5202b61c2e68541469cc6d045 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Dettlaff?= Date: Mon, 19 Feb 2024 22:41:35 +0100 Subject: [PATCH 1/3] support setting factories Allow the settings objects imported during `saq.worker.start()` to be callable. This makes it possible to eliminate import-time side effects, especially if creating the settings objects is more involved. --- saq/worker.py | 7 ++++- tests/test_worker.py | 69 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/saq/worker.py b/saq/worker.py index 5d403a2..7f4f7cd 100644 --- a/saq/worker.py +++ b/saq/worker.py @@ -312,7 +312,12 @@ def import_settings(settings: str) -> dict[str, t.Any]: # given a.b.c, parses out a.b as the module path and c as the variable module_path, name = settings.strip().rsplit(".", 1) module = importlib.import_module(module_path) - return getattr(module, name) + settings_obj = getattr(module, name) + + if callable(settings_obj): + settings_obj = settings_obj() + + return settings_obj def start( diff --git a/tests/test_worker.py b/tests/test_worker.py index a417e1e..d3f45fd 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -2,14 +2,20 @@ import asyncio import contextvars +import contextlib import logging +import secrets +import sys +import tempfile +import textwrap import typing as t import unittest from unittest import mock +from pathlib import Path from saq.job import CronJob, Job, Status from saq.utils import uuid1 -from saq.worker import Worker +from saq.worker import Worker, import_settings from tests.helpers import cleanup_queue, create_queue if t.TYPE_CHECKING: @@ -319,3 +325,64 @@ async def before_enqueue(job: Job) -> None: correlation_ids = await self.queue.apply("recurse", n=2) self.assertEqual(len(correlation_ids), 3) self.assertTrue(all(cid == correlation_ids[0] for cid in correlation_ids[1:])) + + +class TestSettingsImport(unittest.TestCase): + def setUp(self) -> None: + self.cm = cm = contextlib.ExitStack() + + tempdir = Path(cm.enter_context(tempfile.TemporaryDirectory())) + root_module_name = "foo" + secrets.token_urlsafe(2) + file_tree = [ + tempdir / root_module_name / "__init__.py", + tempdir / root_module_name / "bar" / "__init__.py", + tempdir / root_module_name / "bar" / "settings.py", + ] + for path in file_tree: + path.parent.mkdir(exist_ok=True, parents=True) + path.touch() + + file_tree[-1].write_text( + textwrap.dedent( + """ + static = { + "functions": ["pretend_its_a_fn"], + "concurrency": 100 + } + + def factory(): + return { + "functions": ["pretend_its_some_other_fn"], + "concurrency": static["concurrency"] + 100 + } + """ + ).strip() + ) + sys.path.append(str(tempdir)) + + self.module_path = f"{root_module_name}.bar.settings" + + def tearDown(self) -> None: + self.cm.close() + + def test_imports_settings_from_module_path(self) -> None: + settings = import_settings(self.module_path + ".static") + + self.assertDictEqual( + settings, + { + "functions": ["pretend_its_a_fn"], + "concurrency": 100, + }, + ) + + def test_calls_settings_factory(self) -> None: + settings = import_settings(self.module_path + ".factory") + + self.assertDictEqual( + settings, + { + "functions": ["pretend_its_some_other_fn"], + "concurrency": 200, + }, + ) From 23a93bda7b88f415a9303d072a091e6f71807967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Dettlaff?= Date: Mon, 19 Feb 2024 23:24:39 +0100 Subject: [PATCH 2/3] update readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f107741..6cee3da 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,8 @@ To start the worker, assuming the previous is available in the python path saq module.file.settings ``` +> **_Note:_** `module.file.settings` can also be a callable returning the settings dictionary. + To enqueue jobs ```python From 7735a27b05550cc64ad5ae0832999a7784a6b704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Dettlaff?= Date: Mon, 19 Feb 2024 23:48:53 +0100 Subject: [PATCH 3/3] move settings import tests to a separate file --- tests/test_settings_import.py | 72 +++++++++++++++++++++++++++++++++++ tests/test_worker.py | 69 +-------------------------------- 2 files changed, 73 insertions(+), 68 deletions(-) create mode 100644 tests/test_settings_import.py diff --git a/tests/test_settings_import.py b/tests/test_settings_import.py new file mode 100644 index 0000000..dbcf7bd --- /dev/null +++ b/tests/test_settings_import.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import contextlib +import secrets +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + +from saq.worker import import_settings + + +class TestSettingsImport(unittest.TestCase): + def setUp(self) -> None: + self.cm = cm = contextlib.ExitStack() + + tempdir = Path(cm.enter_context(tempfile.TemporaryDirectory())) + root_module_name = "foo" + secrets.token_urlsafe(2) + file_tree = [ + tempdir / root_module_name / "__init__.py", + tempdir / root_module_name / "bar" / "__init__.py", + tempdir / root_module_name / "bar" / "settings.py", + ] + for path in file_tree: + path.parent.mkdir(exist_ok=True, parents=True) + path.touch() + + file_tree[-1].write_text( + textwrap.dedent( + """ + static = { + "functions": ["pretend_its_a_fn"], + "concurrency": 100 + } + + def factory(): + return { + "functions": ["pretend_its_some_other_fn"], + "concurrency": static["concurrency"] + 100 + } + """ + ).strip() + ) + sys.path.append(str(tempdir)) + + self.module_path = f"{root_module_name}.bar.settings" + + def tearDown(self) -> None: + self.cm.close() + + def test_imports_settings_from_module_path(self) -> None: + settings = import_settings(self.module_path + ".static") + + self.assertDictEqual( + settings, + { + "functions": ["pretend_its_a_fn"], + "concurrency": 100, + }, + ) + + def test_calls_settings_factory(self) -> None: + settings = import_settings(self.module_path + ".factory") + + self.assertDictEqual( + settings, + { + "functions": ["pretend_its_some_other_fn"], + "concurrency": 200, + }, + ) diff --git a/tests/test_worker.py b/tests/test_worker.py index d3f45fd..a417e1e 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -2,20 +2,14 @@ import asyncio import contextvars -import contextlib import logging -import secrets -import sys -import tempfile -import textwrap import typing as t import unittest from unittest import mock -from pathlib import Path from saq.job import CronJob, Job, Status from saq.utils import uuid1 -from saq.worker import Worker, import_settings +from saq.worker import Worker from tests.helpers import cleanup_queue, create_queue if t.TYPE_CHECKING: @@ -325,64 +319,3 @@ async def before_enqueue(job: Job) -> None: correlation_ids = await self.queue.apply("recurse", n=2) self.assertEqual(len(correlation_ids), 3) self.assertTrue(all(cid == correlation_ids[0] for cid in correlation_ids[1:])) - - -class TestSettingsImport(unittest.TestCase): - def setUp(self) -> None: - self.cm = cm = contextlib.ExitStack() - - tempdir = Path(cm.enter_context(tempfile.TemporaryDirectory())) - root_module_name = "foo" + secrets.token_urlsafe(2) - file_tree = [ - tempdir / root_module_name / "__init__.py", - tempdir / root_module_name / "bar" / "__init__.py", - tempdir / root_module_name / "bar" / "settings.py", - ] - for path in file_tree: - path.parent.mkdir(exist_ok=True, parents=True) - path.touch() - - file_tree[-1].write_text( - textwrap.dedent( - """ - static = { - "functions": ["pretend_its_a_fn"], - "concurrency": 100 - } - - def factory(): - return { - "functions": ["pretend_its_some_other_fn"], - "concurrency": static["concurrency"] + 100 - } - """ - ).strip() - ) - sys.path.append(str(tempdir)) - - self.module_path = f"{root_module_name}.bar.settings" - - def tearDown(self) -> None: - self.cm.close() - - def test_imports_settings_from_module_path(self) -> None: - settings = import_settings(self.module_path + ".static") - - self.assertDictEqual( - settings, - { - "functions": ["pretend_its_a_fn"], - "concurrency": 100, - }, - ) - - def test_calls_settings_factory(self) -> None: - settings = import_settings(self.module_path + ".factory") - - self.assertDictEqual( - settings, - { - "functions": ["pretend_its_some_other_fn"], - "concurrency": 200, - }, - )