From b7f7b022c731fc38778b8335d630a0b99bb718fd Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 1 Feb 2022 10:05:18 -0600 Subject: [PATCH 1/5] enable specifying keys to populate --- datajoint/autopopulate.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/datajoint/autopopulate.py b/datajoint/autopopulate.py index baf2284ff..ec0ab31e4 100644 --- a/datajoint/autopopulate.py +++ b/datajoint/autopopulate.py @@ -19,7 +19,7 @@ def _initialize_populate(table, jobs, populate_kwargs): """ - Initialize the process for mulitprocessing. + Initialize the process for multiprocessing. Saves the unpickled copy of the table to the current process and reconnects. """ process = mp.current_process() @@ -126,7 +126,8 @@ def _jobs_to_do(self, restrictions): pass return (todo & AndList(restrictions)).proj() - def populate(self, *restrictions, suppress_errors=False, return_exception_objects=False, + def populate(self, *restrictions, keys=None, suppress_errors=False, + return_exception_objects=False, reserve_jobs=False, order="original", limit=None, max_calls=None, display_progress=False, processes=1): """ @@ -134,6 +135,8 @@ def populate(self, *restrictions, suppress_errors=False, return_exception_object for which there is not already a tuple in table. :param restrictions: a list of restrictions each restrict (table.key_source - target.proj()) + :param keys: The list of dicts to populate. When None (default), + uses self.key_source to query keys to populate. :param suppress_errors: if True, do not terminate execution. :param return_exception_objects: return error objects instead of just error messages :param reserve_jobs: if True, reserve jobs to populate in asynchronous fashion @@ -159,7 +162,8 @@ def handler(signum, frame): raise SystemExit('SIGTERM received') old_handler = signal.signal(signal.SIGTERM, handler) - keys = (self._jobs_to_do(restrictions) - self.target).fetch("KEY", limit=limit) + keys = keys if keys is not None else \ + (self._jobs_to_do(restrictions) - self.target).fetch("KEY", limit=limit) if order == "reverse": keys.reverse() elif order == "random": @@ -216,6 +220,7 @@ def _populate1(self, key, jobs, suppress_errors, return_exception_objects): :param return_exception_objects: if True, errors must be returned as objects :return: (key, error) when suppress_errors=True, otherwise None """ + # use the legacy `_make_tuples` callback. make = self._make_tuples if hasattr(self, '_make_tuples') else self.make if jobs is None or jobs.reserve(self.target.table_name, self._job_key(key)): From 55d1006f4ebd91201ca6f03b5ce80fa38153590e Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Tue, 1 Feb 2022 10:19:15 -0600 Subject: [PATCH 2/5] minor cosmetic --- datajoint/autopopulate.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datajoint/autopopulate.py b/datajoint/autopopulate.py index ec0ab31e4..4d9386b6b 100644 --- a/datajoint/autopopulate.py +++ b/datajoint/autopopulate.py @@ -162,8 +162,9 @@ def handler(signum, frame): raise SystemExit('SIGTERM received') old_handler = signal.signal(signal.SIGTERM, handler) - keys = keys if keys is not None else \ - (self._jobs_to_do(restrictions) - self.target).fetch("KEY", limit=limit) + if keys is None: + keys = (self._jobs_to_do(restrictions) - self.target).fetch( + "KEY", limit=limit) if order == "reverse": keys.reverse() elif order == "random": From 395b1d850bf9a3d7624ee9583997eedcb6f9e200 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Wed, 9 Feb 2022 20:17:48 -0600 Subject: [PATCH 3/5] improve docstring --- datajoint/autopopulate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datajoint/autopopulate.py b/datajoint/autopopulate.py index 4d9386b6b..0fa03a716 100644 --- a/datajoint/autopopulate.py +++ b/datajoint/autopopulate.py @@ -135,8 +135,8 @@ def populate(self, *restrictions, keys=None, suppress_errors=False, for which there is not already a tuple in table. :param restrictions: a list of restrictions each restrict (table.key_source - target.proj()) - :param keys: The list of dicts to populate. When None (default), - uses self.key_source to query keys to populate. + :param keys: The list of keys (dicts) to send to self.make(). + If None (default), then use self.key_source to query they keys. :param suppress_errors: if True, do not terminate execution. :param return_exception_objects: return error objects instead of just error messages :param reserve_jobs: if True, reserve jobs to populate in asynchronous fashion From 010e7d7047d58a13642d58e9e82e482457ee48ea Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Sun, 29 May 2022 22:02:56 -0500 Subject: [PATCH 4/5] fixed message when deleting with transactions off --- datajoint/table.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/datajoint/table.py b/datajoint/table.py index 45b2c7284..1572feb3a 100644 --- a/datajoint/table.py +++ b/datajoint/table.py @@ -583,17 +583,16 @@ def cascade(table): print("Nothing to delete.") if transaction: self.connection.cancel_transaction() + elif not transaction: + print("Delete completed") + elif not safemode or user_choice("Commit deletes?", default="no") == "yes": + self.connection.commit_transaction() + if safemode: + print("Deletes committed.") else: - if not safemode or user_choice("Commit deletes?", default="no") == "yes": - if transaction: - self.connection.commit_transaction() - if safemode: - print("Deletes committed.") - else: - if transaction: - self.connection.cancel_transaction() - if safemode: - print("Deletes cancelled") + self.connection.cancel_transaction() + if safemode: + print("Deletes cancelled") return delete_count def drop_quick(self): From b85c9592227081a6e804a4a964ff44b320567bb0 Mon Sep 17 00:00:00 2001 From: Dimitri Yatsenko Date: Thu, 12 Sep 2024 13:27:29 -0500 Subject: [PATCH 5/5] black formatting --- datajoint/autopopulate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datajoint/autopopulate.py b/datajoint/autopopulate.py index debe78b22..0e16ee29b 100644 --- a/datajoint/autopopulate.py +++ b/datajoint/autopopulate.py @@ -211,7 +211,8 @@ def handler(signum, frame): if keys is None: keys = (self._jobs_to_do(restrictions) - self.target).fetch( - "KEY", limit=limit) + "KEY", limit=limit + ) # exclude "error", "ignore" or "reserved" jobs if reserve_jobs: