From dcfc621a66769eb5f72d2c89f6e217e1b3029a81 Mon Sep 17 00:00:00 2001 From: "Richard Kuo (Danswer)" Date: Wed, 8 Jan 2025 09:43:47 -0800 Subject: [PATCH 1/3] add more debug logging for locking issue --- .../background/celery/tasks/indexing/tasks.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/backend/onyx/background/celery/tasks/indexing/tasks.py b/backend/onyx/background/celery/tasks/indexing/tasks.py index 8443bb1f079..665165278c6 100644 --- a/backend/onyx/background/celery/tasks/indexing/tasks.py +++ b/backend/onyx/background/celery/tasks/indexing/tasks.py @@ -298,6 +298,15 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None: if search_settings_instance.id == search_settings_list[0].id: search_settings_primary = True + if tenant_id in debug_tenants: + ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK) + task_logger.info( + f"check_for_indexing cc_pair should index: " + f"tenant={tenant_id} " + f"cc_pair={cc_pair_id} " + f"ttl={ttl}" + ) + if not _should_index( cc_pair=cc_pair, last_index=last_attempt, @@ -326,6 +335,15 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None: cc_pair.id, None, db_session ) + if tenant_id in debug_tenants: + ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK) + task_logger.info( + f"check_for_indexing cc_pair try_creating_indexing_task: " + f"tenant={tenant_id} " + f"cc_pair={cc_pair_id} " + f"ttl={ttl}" + ) + # using a task queue and only allowing one task per cc_pair/search_setting # prevents us from starving out certain attempts attempt_id = try_creating_indexing_task( @@ -346,6 +364,15 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None: ) tasks_created += 1 + if tenant_id in debug_tenants: + ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK) + task_logger.info( + f"check_for_indexing cc_pair try_creating_indexing_task finished: " + f"tenant={tenant_id} " + f"cc_pair={cc_pair_id} " + f"ttl={ttl}" + ) + # debugging logic - remove after we're done if tenant_id in debug_tenants: ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK) From a22d02ff7050d8adeeefead4416a9b47e8898e86 Mon Sep 17 00:00:00 2001 From: "Richard Kuo (Danswer)" Date: Wed, 8 Jan 2025 10:01:24 -0800 Subject: [PATCH 2/3] add another log line --- backend/onyx/background/celery/tasks/indexing/tasks.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/onyx/background/celery/tasks/indexing/tasks.py b/backend/onyx/background/celery/tasks/indexing/tasks.py index 665165278c6..09542e74694 100644 --- a/backend/onyx/background/celery/tasks/indexing/tasks.py +++ b/backend/onyx/background/celery/tasks/indexing/tasks.py @@ -278,6 +278,15 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None: db_session ) for search_settings_instance in search_settings_list: + if tenant_id in debug_tenants: + ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK) + task_logger.info( + f"check_for_indexing cc_pair search settings lock: " + f"tenant={tenant_id} " + f"cc_pair={cc_pair_id} " + f"ttl={ttl}" + ) + redis_connector_index = redis_connector.new_index( search_settings_instance.id ) From 1f48de9731458b6294807da28ffc91d4638b96a3 Mon Sep 17 00:00:00 2001 From: "Richard Kuo (Danswer)" Date: Wed, 8 Jan 2025 12:49:24 -0800 Subject: [PATCH 3/3] more logging --- .../background/celery/tasks/indexing/tasks.py | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/backend/onyx/background/celery/tasks/indexing/tasks.py b/backend/onyx/background/celery/tasks/indexing/tasks.py index 09542e74694..9fd73972d0e 100644 --- a/backend/onyx/background/celery/tasks/indexing/tasks.py +++ b/backend/onyx/background/celery/tasks/indexing/tasks.py @@ -293,20 +293,34 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None: if redis_connector_index.fenced: continue + if tenant_id in debug_tenants: + ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK) + task_logger.info( + f"check_for_indexing get_connector_credential_pair_from_id: " + f"tenant={tenant_id} " + f"cc_pair={cc_pair_id} " + f"ttl={ttl}" + ) + cc_pair = get_connector_credential_pair_from_id( cc_pair_id, db_session ) if not cc_pair: continue + if tenant_id in debug_tenants: + ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK) + task_logger.info( + f"check_for_indexing get_last_attempt_for_cc_pair: " + f"tenant={tenant_id} " + f"cc_pair={cc_pair_id} " + f"ttl={ttl}" + ) + last_attempt = get_last_attempt_for_cc_pair( cc_pair.id, search_settings_instance.id, db_session ) - search_settings_primary = False - if search_settings_instance.id == search_settings_list[0].id: - search_settings_primary = True - if tenant_id in debug_tenants: ttl = redis_client.ttl(OnyxRedisLocks.CHECK_INDEXING_BEAT_LOCK) task_logger.info( @@ -316,6 +330,10 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None: f"ttl={ttl}" ) + search_settings_primary = False + if search_settings_instance.id == search_settings_list[0].id: + search_settings_primary = True + if not _should_index( cc_pair=cc_pair, last_index=last_attempt,