From a307399846a3d1b45a3d3cb4d24b20c08374b5f2 Mon Sep 17 00:00:00 2001 From: Sergiy Matusevych Date: Wed, 3 Jul 2024 16:40:47 -0700 Subject: [PATCH 1/2] bugfix: MySQL can round microseconds into the future causing scheduler to skip trials. truncate microseconds downwards when scheduling new trials to fix that --- mlos_bench/mlos_bench/storage/sql/experiment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mlos_bench/mlos_bench/storage/sql/experiment.py b/mlos_bench/mlos_bench/storage/sql/experiment.py index 721971bfeb4..a56968c3327 100644 --- a/mlos_bench/mlos_bench/storage/sql/experiment.py +++ b/mlos_bench/mlos_bench/storage/sql/experiment.py @@ -253,7 +253,8 @@ def _get_config_id(self, conn: Connection, tunables: TunableGroups) -> int: def new_trial(self, tunables: TunableGroups, ts_start: Optional[datetime] = None, config: Optional[Dict[str, Any]] = None) -> Storage.Trial: - ts_start = utcify_timestamp(ts_start or datetime.now(UTC), origin="local") + # bugfix: MySQL can round microseconds into the future causing scheduler to skip trials. + ts_start = utcify_timestamp(ts_start or datetime.now(UTC), origin="local").replace(microsecond=0) _LOG.debug("Create trial: %s:%d @ %s", self._experiment_id, self._trial_id, ts_start) with self._engine.begin() as conn: try: From 208e33c00efc3d9ec10f60b8997a54ea21cabff5 Mon Sep 17 00:00:00 2001 From: Sergiy Matusevych Date: Wed, 3 Jul 2024 16:49:41 -0700 Subject: [PATCH 2/2] better wording in the comment --- mlos_bench/mlos_bench/storage/sql/experiment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mlos_bench/mlos_bench/storage/sql/experiment.py b/mlos_bench/mlos_bench/storage/sql/experiment.py index a56968c3327..108d3e1d2de 100644 --- a/mlos_bench/mlos_bench/storage/sql/experiment.py +++ b/mlos_bench/mlos_bench/storage/sql/experiment.py @@ -253,7 +253,8 @@ def _get_config_id(self, conn: Connection, tunables: TunableGroups) -> int: def new_trial(self, tunables: TunableGroups, ts_start: Optional[datetime] = None, config: Optional[Dict[str, Any]] = None) -> Storage.Trial: - # bugfix: MySQL can round microseconds into the future causing scheduler to skip trials. + # MySQL can round microseconds into the future causing scheduler to skip trials. + # Truncate microseconds to avoid this issue. ts_start = utcify_timestamp(ts_start or datetime.now(UTC), origin="local").replace(microsecond=0) _LOG.debug("Create trial: %s:%d @ %s", self._experiment_id, self._trial_id, ts_start) with self._engine.begin() as conn: